def testAuth(self):
ft = firethorn.Firethorn(endpoint=firethorn.config.endpoint)
ft.login(firethorn.config.adminuser, firethorn.config.adminpass,
firethorn.config.admingroup)
# Create a JdbcResource to represent the local JDBC database.
jdbc_name = "ATLAS JDBC resource"
atlas_jdbc = ft.firethorn_engine.create_jdbc_resource(
jdbc_name, firethorn.config.datadata, firethorn.config.datacatalog,
firethorn.config.datatype, firethorn.config.datahost,
firethorn.config.datauser, firethorn.config.datapass)
# Locate the JdbcSchema based on catalog and schema name.
catalog = "ATLASDR1"
schema = "dbo"
# Jdbc Schema Tests
atlas_jdbc_schema = ft.firethorn_engine.select_jdbc_schema_by_name(
atlas_jdbc.url, catalog, schema)
# Create an empty AdqlResource to represent the local JDBC database.
adqlname = "ATLAS ADQL resource"
atlas_adql = ft.firethorn_engine.create_adql_resource(adqlname)
# Import the mapping between JDBC and ADQL tables.
metadoc = "https://raw.githubusercontent.com/stvoutsin/firethorn.py/master/firethorn/meta/ATLASDR1_TablesSchema.xml"
atlas_adql_schema = atlas_adql.import_jdbc_schema(atlas_jdbc_schema,
metadoc=metadoc)
# Create an empty AdqlResource to represent a Query Resource
adqlname = "Query resource"
query_resource = ft.firethorn_engine.create_adql_resource(adqlname)
query_resource.import_adql_schema(atlas_adql_schema)
querytext = "SELECT * FROM ATLASDR1.Filter"
print("Creating query using AdqlQuery.. ")
admin_query = query_resource.create_query(querytext)
print(admin_query)
print("List of Running queries: ")
print(query_resource.select_queries())
admin_query = admin_query.update(adql_query_status_next="COMPLETED")
while admin_query.isRunning():
print(admin_query.status())
time.sleep(5)
print(admin_query)
print(admin_query.results())
def testAuth(self):
ft = firethorn.Firethorn(endpoint=firethorn.config.endpoint)
ft.login(firethorn.config.adminuser, firethorn.config.adminpass,
firethorn.config.admingroup)
# Get list of Public (WFAU) Resources
ft.get_public_resource_names()
# Select a Resource by name
osa_resource = ft.get_public_resource_by_name('OSA')
# Get list of Schemas for OSA resource
osa_resource.get_schema_names()
atlasdr1 = osa_resource.get_schema_by_name("ATLASDR1")
# Get list of Tables for ATLASDR1 Schema
atlasdr1.get_table_names()
atlasSource = atlasdr1.get_table_by_name("atlasSource")
# Get list of Columns for atlasSource Table
atlasSource.get_column_names()
ra = atlasSource.get_column_by_name("ra")
# Create new workspace
my_workspace = ft.new_workspace("mySpace")
# Import a Schema from the OSA workspace
my_workspace.add_schema(osa_resource.get_schema_by_name("ATLASDR1"))
# List the workspace schema.
print(my_workspace.get_schemas())
# List the workspace schema.
my_workspace.get_schema_names()
# Run an Asynchronous query
async_query = my_workspace.query("SELECT TOP * FROM ATLASDR1.Filter",
"ASYNC")
# Test an Asynchronous query
print(async_query.error())
async_query.update("SELECT TOP 1000 * FROM ATLASDR1.atlasSource")
async_query.run()
while async_query.isRunning():
print(async_query.status())
time.sleep(5)
print(async_query.results().as_astropy())
# Run a Synchronous query
query = my_workspace.query(
"SELECT TOP 1000 * FROM ATLASDR1.atlasSource")
# Get results table
table = query.results()
print("Status: " + str(query.status()))
# Get results table row count
print("Row count: " + str(table.rowcount()))
# Get results table as astropy table
table.as_astropy()
import firethorn_utils.validator as validator
import firethorn
from argparse import ArgumentParser
if __name__ == "__main__":
'''
Validator Arguments
'''
parser = ArgumentParser()
parser.add_argument("-ft", "--firethorn_url", dest="firethorn_url",
help="Firethorn Instance URL", metavar="Firethorn")
parser.add_argument("-r", "--resource_id", dest="resource_id",
help="Resource ID", metavar="RESOURCE")
parser.add_argument("-u", "--username", dest="username",
help="Firethorn username", metavar="USERNAME")
parser.add_argument("-p", "--pass", dest="password",
help="Firethorn password", metavar="PASSWORD")
parser.add_argument("-g", "--group", dest="group",
help="Firethorn group", metavar="GROUP")
parser.add_argument("-v", "--verbose", dest="verbose", default=True,
help="Print status messages to stdout")
args = parser.parse_args()
ft = firethorn.Firethorn(endpoint=args.firethorn_url)
ft.login(args.username, args.password, args.group)
validator_obj = validator.Validator(ft, args.verbose)
validator_obj.validate(args.firethorn_url + "/adql/resource/" + args.resource_id)
def testSetupTap(self):
with open('../data/osa-tap.json') as json_data:
json_obect = json.load(json_data)
name = json_obect.get("name")
databases = json_obect.get("Databases")
jdbc_resource = json_obect.get("JdbcResource")
metadata = json_obect.get("Metadata")
ft = firethorn.Firethorn(
endpoint="http://gworewia.metagrid.xyz/firethorn")
ft.login(firethorn.config.adminuser, firethorn.config.adminpass,
firethorn.config.admingroup)
# Create a JdbcResource to represent the local JDBC database.
jdbc_name = name + " JDBC resource"
tap_jdbc = ft.firethorn_engine.create_jdbc_resource(
jdbc_name, jdbc_resource["datadata"], jdbc_resource["datacatalog"],
jdbc_resource["datatype"], jdbc_resource["datahost"],
jdbc_resource["datauser"], jdbc_resource["datapass"])
print(tap_jdbc)
# Create an empty AdqlResource to represent the local JDBC database.
tap_name = name + " ADQL resource"
tap_adql = ft.firethorn_engine.create_adql_resource(tap_name)
for database in databases:
jdbc_schema = tap_jdbc.select_schema_by_name(
database.get("name"), "dbo")
if (jdbc_schema != None):
metadoc = "https://raw.githubusercontent.com/wfau/metadata/master/metadocs/" + database.get(
"name") + "_TablesSchema.xml"
adql_schema = tap_adql.import_jdbc_schema(jdbc_schema,
database.get("name"),
metadoc=metadoc)
print(adql_schema)
print(tap_adql)
tap_schema_user = metadata["user"]
tap_schema_pass = metadata["pass"]
tap_schema_url = metadata["url"]
tap_schema_driver = "net.sourceforge.jtds.jdbc.Driver"
tap_schema_db = metadata["database"]
data = urllib.parse.urlencode({
"url": tap_schema_url,
"user": tap_schema_user,
"pass": tap_schema_pass,
"driver": tap_schema_driver,
"catalog": tap_schema_db
}).encode("utf-8")
# Generate TAP_SCHEMA
req = urllib.request.Request(
ft.endpoint + "/tap/" + tap_adql.ident() + "/generateTapSchema",
headers=tap_adql.account.get_identity_as_headers())
response = urllib.request.urlopen(req, data)
response.close()
print(ft.endpoint + "/tap/" + tap_adql.ident())
For every AdqlResource in the json_file, setup the Resources (and TAP service if tap_included=true)
"""
if (self.json_file.lower().startswith("http")):
with urllib.request.urlopen(self.json_file) as url:
json_obect = json.loads(url.read().decode())
else:
data = open(self.json_file)
json_obect = json.load(data)
name = json_obect.get("name")
adql_resources_json = json_obect.get("AdqlResources")
jdbc_resources_json = json_obect.get("JdbcResources")
self.load_jdbc_resources(jdbc_resources_json)
for resource in adql_resources_json:
new_adql_resource = self.create_adql_resource(resource)
if (self.tap_included):
tap = self.create_tap_service(new_adql_resource)
print("TAP Service available at: " + tap)
print("")
if __name__ == "__main__":
ft = firethorn.Firethorn(endpoint=firethorn.config.endpoint)
ft.login(firethorn.config.adminuser, firethorn.config.adminpass,
firethorn.config.admingroup)
ft.firethorn_engine.load_resources("../data/vsa-tap.json")
#sEng = SetupEngine(json_file="https://raw.githubusercontent.com/stvoutsin/firethorn.py/dev/firethorn/data/osa-tap.json", firethorn_base="http://localhost:8081/firethorn")
#sEng.setup_resources()
def testAuth(self):
ft = firethorn.Firethorn(endpoint=firethorn.config.endpoint)
print("Checking System info as " + str(ft.identity()))
print(ft.firethorn_engine.identity())
print(ft.firethorn_engine.system_info_check())
print("")
print("Checking System info as " + str(ft.identity()))
ft.firethorn_engine.account.community = None
print(ft.firethorn_engine.identity())
print(ft.firethorn_engine.system_info_check())
print("")
print("Checking System info as " + str(ft.identity()))
ft.firethorn_engine.account.password = ""
print(ft.firethorn_engine.identity())
print(ft.firethorn_engine.system_info_check())
print("")
ft.login("orinoco", "wombleden", "wombles")
print("Checking System info as " + str(ft.identity()))
print(ft.firethorn_engine.identity())
print(ft.firethorn_engine.system_info_check())
print("")
ft.firethorn_engine.account.community = "NOT-wombles"
print("Checking System info as " + str(ft.identity()))
print(ft.firethorn_engine.identity())
print(ft.firethorn_engine.system_info_check())
print("")
ft.login("orinoco", "wombleden", "wombles")
ft.firethorn_engine.account.password = "******"
print("Checking System info as " + str(ft.identity()))
print(ft.firethorn_engine.identity())
print(ft.firethorn_engine.system_info_check())
print("")
ft.login("orinoco", "NOT-wombleden", "wombles")
print("Checking System info as " + str(ft.identity()))
print(ft.firethorn_engine.identity())
print(ft.firethorn_engine.system_info_check())
print("")
ft.login("*****@*****.**", "password")
print("Checking System info as " + str(ft.identity()))
print(ft.firethorn_engine.identity())
print(ft.firethorn_engine.system_info_check())
print("")
ft = firethorn.Firethorn(endpoint=firethorn.config.default_endpoint +
"/firethorn")
print("Try creating JDBC resources as " + str(ft.identity()))
print(ft.firethorn_engine.identity())
# Create a JdbcResource to represent the local JDBC database.
jdbc_name = "ATLAS JDBC resource"
jdbc_url = "jdbc:jtds:sqlserver://" + firethorn.config.datahost + "/ATLASDR1"
atlas_jdbc_url = ft.firethorn_engine.create_jdbc_resource(
jdbc_name, firethorn.config.datadata, firethorn.config.datacatalog,
firethorn.config.datatype, firethorn.config.datahost,
firethorn.config.datauser, firethorn.config.datapass)
print(atlas_jdbc_url)
print("")
ft.login("orinoco", "wombleden", "wombles")
print("Try creating JDBC resources as " + str(ft.identity()))
print(ft.firethorn_engine.identity())
print("")
# Create a JdbcResource to represent the local JDBC database.
jdbc_name = "ATLAS JDBC resource"
jdbc_url = "jdbc:jtds:sqlserver://" + firethorn.config.datahost + "/ATLASDR1"
atlas_jdbc_url = ft.firethorn_engine.create_jdbc_resource(
jdbc_name, firethorn.config.datadata, firethorn.config.datacatalog,
firethorn.config.datatype, firethorn.config.datahost,
firethorn.config.datauser, firethorn.config.datapass)
print(atlas_jdbc_url)
print("")
ft.firethorn_engine.create_temporary_account()
print("Select JDBC Resource as ")
print(ft.firethorn_engine.identity())
print(ft.firethorn_engine.get_json(atlas_jdbc_url.url))
print("")
print("Create ADQL Resource as ")
print(ft.firethorn_engine.identity())
adqlname = "ATLAS ADQL resource"
atlas_adql_url = ft.firethorn_engine.create_adql_resource(adqlname)
print(atlas_adql_url)
print("")
print("Select ADQL Resource as ")
print(ft.firethorn_engine.identity())
print(ft.firethorn_engine.get_json(atlas_adql_url.url))
print("")
ft.firethorn_engine.create_temporary_account()
print("Select ADQL Resource as ")
print(ft.firethorn_engine.identity())
print(ft.firethorn_engine.get_json(atlas_adql_url.url))
print("")
pass
def main():
'''
Validator Arguments
'''
parser = ArgumentParser()
parser.add_argument("-ft",
"--firethorn_url",
dest="firethorn_url",
help="Firethorn Instance URL",
metavar="FIRETHORN")
parser.add_argument("-r",
"--resource_id",
dest="resource_id",
help="Resource ID",
metavar="RESOURCE")
parser.add_argument("-u",
"--username",
dest="username",
help="Firethorn username",
metavar="USERNAME")
parser.add_argument("-p",
"--pass",
dest="password",
help="Firethorn password",
metavar="PASSWORD")
parser.add_argument("-g",
"--group",
dest="group",
help="Firethorn group",
metavar="GROUP")
parser.add_argument("-v",
"--verbose",
dest="verbose",
help="Print status messages to stdout")
parser.add_argument("-from",
"--from",
dest="from_email",
help="Email from which to send Validation email",
metavar="FROM")
parser.add_argument("-to",
"--to",
dest="to_email",
help="Email to which to send Validation email",
metavar="TO")
parser.add_argument("-m",
"--mode",
dest="mode",
help="TAP query mode (sync|async)",
metavar="MODE")
args = parser.parse_args()
ft = firethorn.Firethorn(endpoint=args.firethorn_url)
ft.login(args.username, args.password, args.group)
validator_obj = TAPValidator(ft, args.verbose)
validator_results = validator_obj.validate(
args.firethorn_url + "/adql/resource/" + args.resource_id,
args.firethorn_url + "/tap/" + args.resource_id,
mode=args.mode)
print("Processed: ")
print(validator_results.processed)
print("------------")
print("Candidates: ")
print(validator_results.candidates)
print("------------")
print("Exceptions: ")
print(validator_results.exceptions)
print("------------")
'''
Send exceptions by email
'''
print(validator_results.exceptions)
if ((len(validator_results.exceptions) > 0) and (args.from_email != None)
and (args.to_email != None)):
print("Sending email with exceptions to: " + args.to_email)
Utility.sendMail(args.from_email, args.to_email,
"Validation Results - Database Exeptions",
json.dumps(validator_results.exceptions))
def testAuth(self):
ft = firethorn.Firethorn(endpoint=firethorn.config.endpoint)
ft.login(firethorn.config.adminuser, firethorn.config.adminpass,
firethorn.config.admingroup)
# Create a JdbcResource to represent the local JDBC database.
jdbc_name = "ATLAS JDBC resource"
atlas_jdbc = ft.firethorn_engine.create_jdbc_resource(
jdbc_name, firethorn.config.datadata, firethorn.config.datacatalog,
firethorn.config.datatype, firethorn.config.datahost,
firethorn.config.datauser, firethorn.config.datapass)
print("atlas_jdbc: " + str(atlas_jdbc))
print("Ident: " + atlas_jdbc.ident())
print("Name: " + atlas_jdbc.name())
print("Owner: " + atlas_jdbc.owner())
print("URL: " + atlas_jdbc.url)
print("select_schemas() : ")
print(atlas_jdbc.select_schemas())
print("select_schema_by_ident(): ")
print(
atlas_jdbc.select_schema_by_ident(
atlas_jdbc.select_schemas()[0].url))
print("select_schema_by_name(): ")
print(atlas_jdbc.select_schema_by_name("ATLASDR1", "dbo"))
print("create_schema(): ")
print(atlas_jdbc.create_schema(
"dbo", "mySchema")) ## ???? Not implemented yet
# Locate the JdbcSchema based on catalog and schema name.
catalog = "ATLASDR1"
schema = "dbo"
print("select_jdbc_resource_by_name(): ")
print(ft.firethorn_engine.select_jdbc_resource_by_name(
catalog)) ## ??? ## ???? Not implemented yet
print("select_jdbc_resource_by_ident(): ")
print(ft.firethorn_engine.select_jdbc_resource_by_ident(
atlas_jdbc.url))
# Jdbc Schema Tests
atlas_jdbc_schema = ft.firethorn_engine.select_jdbc_schema_by_name(
atlas_jdbc.url, catalog, schema)
filter_jdbc_table = atlas_jdbc_schema.select_table_by_ident(
atlas_jdbc_schema.select_tables()[0].url)
print("atlas_jdbc_schema: ")
print(atlas_jdbc_schema)
print("resource(): ")
print(atlas_jdbc_schema.resource())
print("catalog_name(): " + atlas_jdbc_schema.catalog_name())
print("select_tables(): ")
print(atlas_jdbc_schema.select_tables())
print("select_table_by_ident(): ")
print(
atlas_jdbc_schema.select_table_by_ident(
atlas_jdbc_schema.select_tables()[0].url))
print("select_table_by_name(): ")
print(atlas_jdbc_schema.select_table_by_name(
filter_jdbc_table.name)) ## ???? ## ???? Not implemented yet
print("create_table(): ")
print(atlas_jdbc_schema.create_table(
"myTable")) ## ???? ## ???? Not implemented yet
# Jdbc Table Tests
print("name(): " + filter_jdbc_table.name())
print("ident(): " + filter_jdbc_table.ident())
print("resource(): ")
print(filter_jdbc_table.resource())
print("schema(): ")
print(filter_jdbc_table.schema())
print("select_columns(): ")
print(filter_jdbc_table.select_columns())
print("select_column_by_ident(): ")
filterID_jdbc_column = filter_jdbc_table.select_column_by_ident(
filter_jdbc_table.select_columns()[0].url)
print(filterID_jdbc_column)
print("select_column_by_name(): ")
print(
filter_jdbc_table.select_column_by_name(
filterID_jdbc_column.name)) ## ?? Not implemented yet
print("create_column(): ")
print(filter_jdbc_table.create_column(
"myColumn")) ## ?? Not implemented yet
# Jdbc Column Tests
print("url: " + filterID_jdbc_column.url)
print("name(): " + filterID_jdbc_column.name())
print("ident(): " + filterID_jdbc_column.ident())
print("owner(): " + filterID_jdbc_column.owner())
print("resource(): ")
print(filterID_jdbc_column.resource())
print("schema(): ")
print(filterID_jdbc_column.schema())
print("table(): ")
print(filterID_jdbc_column.table())
print("type(): " + filterID_jdbc_column.type())
print("arraysize(): " + str(filterID_jdbc_column.size()))
print("ucd(): " + str(filterID_jdbc_column.ucd()))
print("utype(): " + str(filterID_jdbc_column.utype()))
# Create an IvoaResource to represent the GAIA TAP resource.
tapname = "GAIA TAP service"
tapurl = "http://gea.esac.esa.int/tap-server/tap"
vosifile = 'https://raw.githubusercontent.com/stvoutsin/firethorn.py/master/firethorn/meta/vosi/gaia/gaia-tableset.xml'
gaia_ivoa_resource = ft.firethorn_engine.create_ivoa_resource(
tapname, tapurl)
gaia_ivoa_resource.import_ivoa_metadoc(vosifile)
print("gaia_ivoa_resource: ")
print(gaia_ivoa_resource)
print(
ft.firethorn_engine.select_ivoa_resource_by_ident(
gaia_ivoa_resource.url))
print(ft.firethorn_engine.select_ivoa_resource_by_name(
None)) ## Not implemented yet
print(gaia_ivoa_resource.select_schemas())
print(
gaia_ivoa_resource.select_schema_by_ident(
gaia_ivoa_resource.select_schemas()[0].url))
print(gaia_ivoa_resource.select_schema_by_name("gaiadr1"))
# Test IVOA Schema
ivoa_schema = gaia_ivoa_resource.select_schema_by_ident(
gaia_ivoa_resource.select_schemas()[0].url)
print(ivoa_schema.name())
print(ivoa_schema.resource())
print(ivoa_schema.schema_name())
print(ivoa_schema.select_tables())
ivoa_table = ivoa_schema.select_table_by_ident(
ivoa_schema.select_tables()[0].url)
print(ivoa_table)
print(ivoa_schema.select_table_by_name(ivoa_table.name))
# Test IVOA Table
print(ivoa_table)
print(ivoa_table.name())
print(ivoa_table.select_columns())
ivoa_column = ivoa_table.select_column_by_ident(
ivoa_table.select_columns()[0].url)
print(ivoa_column)
print(ivoa_table.select_column_by_name(ivoa_column.name))
# Create an empty AdqlResource to represent the local JDBC database.
adqlname = "ATLAS ADQL resource"
atlas_adql = ft.firethorn_engine.create_adql_resource(adqlname)
print("atlas_adql: ")
print(atlas_adql)
print(ft.firethorn_engine.select_adql_resource_by_ident(
atlas_adql.url))
print(ft.firethorn_engine.select_adql_resource_by_name(
"ATLAS")) ## ?? Not implemented yet
# Import the mapping between JDBC and ADQL tables.
metadoc = "https://raw.githubusercontent.com/stvoutsin/firethorn.py/master/firethorn/meta/ATLASDR1_TablesSchema.xml"
atlas_adql_schema = atlas_adql.import_jdbc_schema(atlas_jdbc_schema,
metadoc=metadoc)
atlas_adql_schemav2 = atlas_adql.import_jdbc_schema(
atlas_jdbc_schema, schema_name="atlasV2")
print(atlas_adql_schema)
print(atlas_adql_schemav2)
print(atlas_adql.select_schema_by_ident(atlas_adql_schema.url))
print(atlas_adql.select_schema_by_name("ATLASDR1"))
print(atlas_adql.select_schemas())
# Create an empty AdqlResource to represent a Query Resource
adqlname = "Query resource"
query_resource = ft.firethorn_engine.create_adql_resource(adqlname)
print("query_resource: ")
print(query_resource)
my_atlas_schema = query_resource.import_adql_schema(
atlas_adql_schema, "myAtlas")
query_resource.import_adql_schema(atlas_adql_schema)
query_resource.import_ivoa_schema(ivoa_schema, "gaiadr1")
print(query_resource.create_adql_schema("mySchema"))
print(query_resource.select_schemas())
# ADQL Schema Tests
print("myAtlas Schema: ")
print(my_atlas_schema)
print("resource(): ")
print(my_atlas_schema.resource())
print("Name: " + my_atlas_schema.name())
print("Owner: " + my_atlas_schema.owner())
print("select_tables(): ")
print(my_atlas_schema.select_tables())
print("select_table_by_ident(): ")
print(
my_atlas_schema.select_table_by_ident(
my_atlas_schema.select_tables()[0].url))
print("select_table_by_name(): ")
my_adql_table = my_atlas_schema.select_table_by_name("Filter")
print(my_adql_table)
print("create_table(): ")
print(my_atlas_schema.create_table(
"myTable")) ## ???? Not implemented yet
my_atlas_schema.import_ivoa_table(None) ## ???? Not implemented yet
my_atlas_schema.import_adql_table(None) ## ???? Not implemented yet
# ADQL Table Tests
print("name(): " + my_adql_table.name())
print("ident(): " + my_adql_table.ident())
print("resource(): ")
print(my_adql_table.resource())
print("schema(): ")
print(my_adql_table.schema())
print("select_columns(): ")
print(my_adql_table.select_columns())
print("select_column_by_ident(): ")
my_column = my_adql_table.select_column_by_ident(
my_adql_table.select_columns()[0].url)
print(my_column)
print("select_column_by_name(): ")
print(my_adql_table.select_column_by_name(
my_column.name)) ## ?? Not implemented yet
print("create_column(): ")
print(my_adql_table.create_adql_column(
"myColumn")) ## ?? Not implemented yet
# ADQL Column Tests
print("url: " + my_column.url)
print("name(): " + my_column.name())
print("ident(): " + my_column.ident())
print("owner(): " + my_column.owner())
print("resource(): ")
print(my_column.resource())
print("schema(): ")
print(my_column.schema())
print("table(): ")
print(my_column.table())
print("type(): " + my_column.type())
print("arraysize(): " + str(my_column.arraysize()))
print("ucd(): " + str(my_column.ucd()))
print("utype(): " + str(my_column.utype()))
# List the workspace schema.
print(atlas_adql.select_schemas())