def action(args):
# Argument validation
if args.directory and not args.output:
sh.die(
"Unable to output to a directory without the output flag (-o)."
)
return
with FailureHandler('get'):
client = Client.build(hostname=args.host)
rs = ResourceSession(client, client.get_session())
rs.get(args.resource, args)
def main():
access_key = subprocess.run(
"credstash -t eng get default_service_account_access_key",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
executable="/bin/bash").stdout.decode('utf-8').strip()
secret_key = subprocess.run(
"credstash -t eng get default_service_account_secret_key",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
executable="/bin/bash").stdout.decode('utf-8').strip()
ascend_client = Client(access_key, secret_key, ENVIRONMENT_PREFIX)
ds = ascend_client.data_service("kitchen_sink")
comp = Component("test",
"small_stuff",
"sources",
"blob",
session=ascend_client.get_session())
print(comp.stream_records())
class ascendClient:
def __init__(self, host="covid19.ascend.io", profile="trial"):
super().__init__()
self.host = host
self.profile = profile
self.client = None
def login(self):
#get Ascend API credentials
access_key, secret_key = ascend_api(self.profile)
self.client = Client(self.host, access_key, secret_key)
print(f"Logged into {self.host}")
# Data service (e.g. COVID_19_Data_Vault) > Data flow (AWS) > Data feed (each box)
def ls(self, data_service=None, data_flow=None):
if not data_service:
print("Listing data services...")
[print(i) for i in self.client.list_data_services()]
elif not data_flow: #specify data_service but not data_flow
print(f"Listing data flows from {data_service}...")
[
print(i) for i in self.client.get_data_service(
data_service).list_dataflows()
]
else:
print(f"Listing components from {data_service}.{data_service}...")
[
print(i) for i in self.client.get_dataflow(
data_service, data_flow).list_components()
]
def get_component(self, data_service, data_flow, component):
return self.client.get_component(data_service, data_flow, component)
def component2pd(self, data_service, data_flow, component):
comp = self.get_component(data_service, data_flow, component)
return pd.DataFrame.from_records(comp.get_records())
def action(args):
with FailureHandler('apply'):
client = Client.build(hostname=args.host)
creds = credentials.load_credentials(args.credentials)
if args.config is not None:
config_file = args.config
try:
with open(config_file, 'r') as f:
args.config = yaml.load(f.read(),
Loader=yaml.SafeLoader)
except Exception as e:
raise Exception(
f'Failure to load config from {config_file}: {e}')
else:
args.config = {}
rs = ResourceSession(client, client.get_session())
rs.apply(args.resource, creds, args)
args = parser.parse_args()
profile = args.profile[0]
host = args.host[0]
ds_list = args.ds_id
config = configparser.ConfigParser()
config.read(os.path.expanduser("~/.ascend/credentials"))
access_id = config.get(profile, "ascend_access_key_id")
secret_key = config.get(profile, "ascend_secret_access_key")
DOCROOT = "./docs/Ascend/"
A = Client(host, access_id, secret_key)
if len(ds_list) == 0:
list_raw = A.list_data_services(raw=True)
ds_list = list(map(lambda d: d["id"], list_raw))
pubs = A.list_data_feeds(raw=True)
pub_by_uuid = dict(
map(lambda p: (p['uuid'], ".".join([p['fromOrgId'], p['id']])), pubs))
pub_name_by_id = dict(
map(lambda p: (".".join([p['fromOrgId'], p['id']]), p['name']), pubs))
def gen_data_service(id):
template = """---
def action(args):
with FailureHandler('list'):
client = Client.build(hostname=args.host)
rs = ResourceSession(client, client.get_session())
rs.list(args.resource, args.recursive)
def action(args):
with FailureHandler('delete'):
client = Client.build(hostname=args.host)
rs = ResourceSession(client, client.get_session())
rs.delete(args.resource, args)
def login(self):
#get Ascend API credentials
access_key, secret_key = ascend_api(self.profile)
self.client = Client(self.host, access_key, secret_key)
print(f"Logged into {self.host}")