def open_session(self):
self.session = Session(self.config.get('flight_action', 'base_url'),
schema=Schema,
request_kwargs={
'auth':
BearerAuth(
self.config.get('flight_action',
'jwt_token'))
})
def connect(self):
models_schema = {
'job_scripts': {
'properties': {
'title': {'type': 'string'},
'language': {'type': 'string'},
'code': {'type': 'string'},
'path': {'type': 'string'},
'defined-at-runtime': {'type': 'boolean'},
'code-strategy': {'type': 'string'}
}
},
'job_templates': {
'properties': {
'title': {'type': 'string'},
'user-params': {
'type': 'object',
'properties': {
'interscity': {
'type': 'object',
'properties': {
'capability': {'type': 'string'},
'sql_queries': {'type': 'string'}
}
}
}
},
'define-schema-at-runtime': {'type': 'boolean'},
'job-script': {'relation': 'to-one', 'resource': ['job-script']}
}
},
'processing_jobs': {
'properties': {
'uuid': {'type': 'string'},
'job-state': {'type': 'string'},
'log': {'type': 'string'},
'job-template': {'relation': 'to-one', 'resource': ['job-template']}
}
}
}
self.session = Session(self.url + "/api", schema=models_schema)
script = self.session.create('job_scripts')
script.title = "SQL Query Script (created by client)"
script.language = "python"
script.code = "empty"
script.path = "interscity_sql_script.py"
script.defined_at_runtime = True
script.code_strategy = "Elixir.DataProcessorBackend.InterSCity.ScriptSamples.SqlQuery"
script.commit()
self.script_id = script.id
def test_list(self, live_server, runs, api_version):
"""
List samples and its metadata for given study
"""
from jsonapi_client import Session
api_base = '%s/%s/' % (live_server.url, api_version)
with Session(api_base) as s:
study = s.get('studies', 'MGYS00000025').resource
assert study.accession == 'MGYS00000025'
assert study.secondary_accession == 'SRP0025'
# list samples
sample_list = study.samples
assert len(sample_list) == 1
for sample in sample_list:
# list runs
run_list = sample.runs
assert len(run_list) == 1
run = run_list[0]
assert study.accession == 'MGYS00000025'
assert study.bioproject == 'PRJDB0025'
assert sample.accession == 'ERS0025'
assert sample.biome.biome_name == 'bar'
assert run.accession == 'ABC_025'
assert run.experiment_type == 'metagenomic'
def main(args=None):
filename = args.filename
with open(filename, "w") as csvfile:
with Session(API_BASE) as s:
fieldnames = [
'study', 'study_name', 'biomes', 'samples_count', 'metadata',
'publications'
]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for study in s.iterate(
'biomes/root:Host-associated:Human:Digestive%20system/studies'
):
biomes = [b.id for b in study.biomes]
publications = "; ".join([p.doi for p in study.publications])
row = {
'study': study.accession,
'study_name': study.study_name,
'biomes': biomes,
'samples_count': study.samples_count,
'publications': publications,
}
metadata = list()
for sample in study.samples:
keys = [m['key'] for m in sample.sample_metadata]
metadata.extend(keys)
row['metadata'] = "; ".join(sorted(set(metadata)))
writer.writerow(row)
def main(args=None):
filename = args.filename
with open(filename, "w") as csvfile:
with Session(API_BASE) as s:
fieldnames = [
"study",
"study_name",
"biomes",
"samples_count",
"metadata",
"publications",
]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for study in s.iterate(
"biomes/root:Host-associated:Human:Digestive%20system/studies"
):
biomes = [b.id for b in study.biomes]
publications = "; ".join([p.doi for p in study.publications])
row = {
"study": study.accession,
"study_name": study.study_name,
"biomes": biomes,
"samples_count": study.samples_count,
"publications": publications,
}
metadata = list()
for sample in study.samples:
keys = [m["key"] for m in sample.sample_metadata]
metadata.extend(keys)
row["metadata"] = "; ".join(sorted(set(metadata)))
writer.writerow(row)
def ebi_sample(sample):
API_BASE = 'https://www.ebi.ac.uk/metagenomics/api/latest/samples'
with Session(API_BASE) as s:
params = {'page_size': 100}
f = Filter(urlencode(params))
sample = s.get(sample).resource
return sample
def main(args=None):
filename = args.filename
accession = args.accession
with open(filename, "w") as csvfile:
with Session(API_BASE) as s:
fieldnames = [
"study",
"sample",
"biome",
"lineage",
"longitude",
"latitude",
]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
study = s.get("studies", accession).resource
for sample in study.samples:
biome = sample.biome
row = {
"study": study.accession,
"sample": sample.accession,
"biome": biome.biome_name,
"lineage": biome.lineage,
"longitude": sample.longitude,
"latitude": sample.latitude,
}
writer.writerow(row)
def get_biome(lineage, exp_type):
API_BASE_BIOME = 'https://www.ebi.ac.uk/metagenomics/api/latest/biomes'
with Session(API_BASE_BIOM) as s:
study = s.get(run, 'analysis').resource
for i in study.downloads:
if extension in i.file_format['name']:
link = i.url
return link
def get_analysis_result(run, extension):
API_BASE_RUN = 'https://www.ebi.ac.uk/metagenomics/api/latest/runs'
with Session(API_BASE_RUN) as s:
study = s.get(run, 'analysis').resource
for i in study.downloads:
if extension in i.file_format['name']:
link = i.url
return link
def test_annotations(self, live_server, run_with_sample, api_version):
from jsonapi_client import Session
call_command('import_summary',
'ABC01234',
os.path.dirname(os.path.abspath(__file__)),
suffix='.go_slim')
api_base = '%s/%s/' % (live_server.url, api_version)
with Session(api_base) as s:
run = s.get('runs', 'ABC01234').resource
for analyses in run.analyses:
go_terms = []
for go in analyses.go_slim:
go_terms.append(go.accession)
expected = ['GO:0030246', 'GO:0046906', 'GO:0043167']
assert go_terms == expected
def main(args=None):
filename = args.filename
with open(filename, "w") as csvfile:
with Session(API_BASE) as s:
fieldnames = ['biome', 'studies_count', 'samples_count']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for biome in s.iterate(
'biomes/root:Host-associated:Human:Digestive%20system/children'
):
studies = s.get('biomes/{}/studies'.format(biome.lineage))
row = {
'biome': biome.id,
'samples_count': biome.samples_count,
'studies_count': studies.meta.pagination['count']
}
writer.writerow(row)
def download_GO_Slim(run_id):
outname = run_id + '_FASTQ_GO_slim.csv'
outdir = 'result_GO_slim'
if not os.path.exists(outdir):
os.mkdir(outdir)
fullname = os.path.join(outdir, outname)
df = DataFrame(columns=('category', 'description', 'annotation counts'))
df.index.name = 'GO term'
API_BASE = 'https://www.ebi.ac.uk/metagenomics/api/latest'
with Session(API_BASE) as s:
run = s.get('runs', run_id).resource
for a in run.analyses:
for ann in a.go_slim:
df.loc[ann.accession] = [
ann.lineage, ann.description, ann.count
]
df = df.rename(columns={'annotation counts': run_id})
return df
def main(args=None):
filename = args.filename
accession = args.accession
with open(filename, "w") as csvfile:
with Session(API_BASE) as s:
fieldnames = ['study', 'sample', 'biome', 'lineage', 'longitude', 'latitude']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
study = s.get('studies', accession).resource
for sample in study.samples:
biome = sample.biome
row = {
'study': study.accession,
'sample': sample.accession,
'biome': biome.biome_name,
'lineage': biome.lineage,
'longitude': sample.longitude,
'latitude': sample.latitude
}
writer.writerow(row)
def main(args=None):
filename = args.filename
with open(filename, "w") as csvfile:
with Session(API_BASE) as s:
fieldnames = ["biome", "studies_count", "samples_count"]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for biome in s.iterate(
"biomes/root:Host-associated:Human:Digestive%20system/children"
):
studies = s.get("biomes/{}/studies".format(biome.lineage))
row = {
"biome": biome.id,
"samples_count": biome.samples_count,
"studies_count": studies.meta.pagination["count"],
}
writer.writerow(row)
# MGYS00001211 (SRP076746) Human gut metagenome Metagenome
# study_accession = "MGYS00001211"
# MGYS00000601 (ERP013908) Assessment of Bacterial DNA Extraction Procedures for Metagenomic Sequencing Evaluated
# on Different Environmental Matrices.
# study_accession = "MGYS00000601"
# MGYS00002115
# The study includes fungal genetic diversity assessment by ITS-1 next generation sequencing (NGS) analyses
# study_accession = "MGYS00002115"
resource = "studies/" + study_accession + "/analyses"
rows = []
with Session(API_BASE) as session:
# TODO: iterate?
analyses = session.get(resource).resources
analyses_accs = [a.accession for a in analyses]
# Select individual analyses, e.g. OSD
# analyses_accs = [""]
for analysis_accession in analyses_accs:
# tax_annotations = .resources
for t in session.iterate("/".join(
["analyses", analysis_accession, "taxonomy", "ssu"])):
if t.hierarchy.get(TAX_RANK):
class ActionApp(App):
"""Action Client primary application."""
class Meta:
# Defines the config paths of the label (among other things)
label = 'flight-action'
# look for configs under the user facing key
config_section = 'flight_action'
# configuration defaults
config_defaults = CONFIG
# call sys.exit() on close
exit_on_close = True
# load additional framework extensions
extensions = ['yaml', 'colorlog', 'mustache']
# configuration handler
config_handler = 'yaml'
# configuration file suffix
config_file_suffix = '.yaml'
# set the log handler
log_handler = 'colorlog'
# set the output handler
output_handler = 'mustache'
# sets the template directory
template_dirs = [join(dirname(__file__), 'templates')]
# register handlers
handlers = [Base]
def __init__(self, *arg, **kwarg):
App.__init__(self, *arg, **kwarg)
self.session = None
def open_session(self):
self.session = Session(self.config.get('flight_action', 'base_url'),
schema=Schema,
request_kwargs={
'auth':
BearerAuth(
self.config.get('flight_action',
'jwt_token'))
})
def close_session(self):
self.session.close()
def add_commands(self):
cmds = self.session.get('commands').resources
for c in cmds:
Base.add_command(c)
# the following hooks are used to interact with the server
# they do the following:
# 1. Establish a session with the server
# 2. Download the commands and adds it to the CLI
# NOTE: This is done each time the command is ran
# *. argparser than runs the action [and assumable uses the session]
# 3. the session is closed
Meta.hooks = [('post_setup', open_session), ('pre_run', add_commands),
('pre_close', close_session)]
class connection():
def __init__(self, url="http://localhost:4000", namespace="api"):
self.url = url
self.namespace = namespace
def connect(self):
models_schema = {
'job_scripts': {
'properties': {
'title': {'type': 'string'},
'language': {'type': 'string'},
'code': {'type': 'string'},
'path': {'type': 'string'},
'defined-at-runtime': {'type': 'boolean'},
'code-strategy': {'type': 'string'}
}
},
'job_templates': {
'properties': {
'title': {'type': 'string'},
'user-params': {
'type': 'object',
'properties': {
'interscity': {
'type': 'object',
'properties': {
'capability': {'type': 'string'},
'sql_queries': {'type': 'string'}
}
}
}
},
'define-schema-at-runtime': {'type': 'boolean'},
'job-script': {'relation': 'to-one', 'resource': ['job-script']}
}
},
'processing_jobs': {
'properties': {
'uuid': {'type': 'string'},
'job-state': {'type': 'string'},
'log': {'type': 'string'},
'job-template': {'relation': 'to-one', 'resource': ['job-template']}
}
}
}
self.session = Session(self.url + "/api", schema=models_schema)
script = self.session.create('job_scripts')
script.title = "SQL Query Script (created by client)"
script.language = "python"
script.code = "empty"
script.path = "interscity_sql_script.py"
script.defined_at_runtime = True
script.code_strategy = "Elixir.DataProcessorBackend.InterSCity.ScriptSamples.SqlQuery"
script.commit()
self.script_id = script.id
def sql(self, queries, capability):
template = self.session.create('job_templates')
template.title = "SQL Query Template - Created by client"
template.user_params = {
'schema': '',
'functional': '',
'interscity': {
'capability': capability,
'sql_queries': queries
}
}
template.define_schema_at_runtime=True
template.job_script_id=self.script_id
template.commit()
template_id = template.id
hd = {'Accept': 'application/vnd.api+json', 'Content-Type': 'application/vnd.api+json'}
endpoint = self.url + "/api/job_templates/{0}/schedule".format(template_id)
r = requests.post(endpoint, headers=hd)
job_id = r.json()['data']['id']
return job_id
def process(self, job_id):
endpoint = self.url + '/api/processing_jobs/{0}/run'.format(job_id)
hd = {'Accept': 'application/vnd.api+json', 'Content-Type': 'application/vnd.api+json'}
print("Running job...")
r = requests.post(endpoint, json={'processing_job_id': job_id}, headers=hd)
print("Done! Response:")
return r
#obtaining study data - TO DO!
def get_biome(lineage, exp_type):
API_BASE_BIOME = 'https://www.ebi.ac.uk/metagenomics/api/latest/biomes'
with Session(API_BASE_BIOM) as s:
study = s.get(run, 'analysis').resource
for i in study.downloads:
if extension in i.file_format['name']:
link = i.url
return link
# In[15]:
API_BASE_BIOME = 'https://www.ebi.ac.uk/metagenomics/api/latest/biomes'
with Session(API_BASE_BIOME) as s:
biome = s.get('root:Host-associated:Human:Skin').resource
# In[21]:
biome.samples_count
# In[22]:
biome.studies
# In[23]:
biome.samples
# In[4]:
def ebi_sample(sample):
API_BASE = 'https://www.ebi.ac.uk/metagenomics/api/latest/samples'
with Session(API_BASE) as s:
sample = s.get(sample).resource
return sample
def get_logbook_records(session, logbook_id, published_since):
path = f'logbooks/{logbook_id}/records'
# This would be better, but we can get by with Modifier until
# we redefine how our endpoints handle filter parameters
# filter = Filter(published_since=published_since)
modifier = Modifier(f'published_since={published_since}')
include = Inclusion('user', 'log')
return session.get(path, modifier + include)
with Session(
LOGCHECK_API_ENDPOINT,
request_kwargs=dict(auth=HTTPBasicAuth(USER_NAME, PASSWORD))) as s:
logbook_id = '2aafb79e4f364feebe6e92393a5aee3d'
published_since = '2019-10-31T00:00:00Z'
document = get_logbook_records(s, logbook_id, published_since)
for record in document.resources:
print(record)
print(record.id)
print(record.value)
print(record.user)
print(record.user.name)
print(record.log.title)
def get_logbook_records(session, logbook_id, published_since):
path = f'logbooks/{logbook_id}/records'
# This would be better, but we can get by with Modifier until
# we redefine how our endpoints handle filter parameters
# filter = Filter(published_since=published_since)
modifier = Modifier(f'published_since={published_since}')
include = Inclusion('user', 'log')
return session.get(path, modifier + include)
with Session(
LOGCHECK_API_ENDPOINT,
request_kwargs=dict(auth=HTTPBasicAuth(TOKEN_ID, SECRET_KEY))) as s:
logbook_id = '2aafb79e4f364feebe6e92393a5aee3d'
published_since = '2019-10-31T00:00:00Z'
document = get_logbook_records(s, logbook_id, published_since)
for record in document.resources:
print(record)
print(record.id)
if 'value' in record._attributes:
print(record.value)
if 'deleted' in record._attributes:
print('(DELETED)')
print(record.user)
print(record.user.name)
def create_app(config_name):
app = Flask(__name__)
# Config
app.config.from_object(config[config_name])
config[config_name].init_app(app)
# Logging
class RequestFormatter(logging.Formatter):
def format(self, record):
record.url = 'NA'
record.request_id = 'NA'
if has_request_context():
record.url = request.url
if app.config['APP_ENABLE_REQUEST_ID']:
record.request_id = request.environ.get("HTTP_X_REQUEST_ID")
return super().format(record)
formatter = RequestFormatter(
'[%(asctime)s] [%(levelname)s] [%(request_id)s] [%(url)s] %(module)s: %(message)s'
)
default_handler.setFormatter(formatter)
default_handler.setLevel(app.config['LOGGING_LEVEL'])
# Middleware / Wrappers
if app.config['APP_ENABLE_PROXY_FIX']:
ReverseProxyPrefixFix(app)
if app.config['APP_ENABLE_REQUEST_ID']:
RequestID(app)
if app.config['APP_ENABLE_SENTRY']:
sentry_sdk.init(**app.config['SENTRY_CONFIG'])
# API client
if not app.config['TESTING']:
arctic_projects_api_auth = BackendApplicationClient(client_id=app.config['AZURE_OAUTH_APPLICATION_ID'])
arctic_projects_api_auth_session = OAuth2Session(client=arctic_projects_api_auth)
arctic_projects_api_auth_token = arctic_projects_api_auth_session.fetch_token(
token_url=app.config['AZURE_OAUTH_TOKEN_URL'],
client_id=app.config['AZURE_OAUTH_APPLICATION_ID'],
client_secret=app.config['AZURE_OAUTH_APPLICATION_SECRET'],
scope=app.config['AZURE_OAUTH_NERC_ARCTIC_OFFICE_SCOPES']
)
arctic_projects_api = Session(
'https://api.bas.ac.uk/arctic-office-projects/testing',
request_kwargs={'headers': {'authorization': f"bearer {arctic_projects_api_auth_token['access_token']}"}})
# Templates
app.jinja_loader = PrefixLoader({
'app': PackageLoader('arctic_office_projects_manager'),
'bas_style_kit': PackageLoader('bas_style_kit_jinja_templates'),
})
app.config['bsk_templates'] = BskTemplates()
app.config['bsk_templates'].site_styles.append({
'href': 'https://cdn.web.bas.ac.uk/libs/font-awesome-pro/5.3.1/css/all.min.css'
})
app.config['bsk_templates'].site_styles.append({
'href': 'https://cdn.rawgit.com/jpswalsh/academicons/master/css/academicons.min.css'
})
app.config['bsk_templates'].site_styles.append({'href': '/static/css/app.css'})
app.config['bsk_templates'].site_title = 'NERC Arctic Office Projects Manager'
app.config['bsk_templates'].bsk_site_nav_brand_text = 'NERC Arctic Office Projects Manager'
app.config['bsk_templates'].site_description = 'Management application for projects in the NERC Arctic Office ' \
'Projects Database'
app.config['bsk_templates'].bsk_site_feedback_href = 'mailto:[email protected]'
app.config['bsk_templates'].bsk_site_nav_primary.append({
'value': 'Projects',
'href': '/projects'
})
app.config['bsk_templates'].bsk_site_nav_launcher.append({
'value': 'Arctic Office Website',
'href': 'https://www.arctic.ac.uk'
})
# Routes
#
@app.route('/')
def index():
return redirect(url_for('projects_index'))
@app.route('/projects')
def projects_index():
projects = arctic_projects_api.get('projects')
# noinspection PyUnresolvedReferences
return render_template(f"app/views/projects_index.j2", projects=projects)
@app.route('/projects/<project_id>')
def project_details(project_id: str):
project = arctic_projects_api.get('projects', project_id)
mermaid_asset = False
for style in app.config['bsk_templates'].site_scripts:
if style['href'] == 'https://unpkg.com/[email protected]/dist/mermaid.min.js':
mermaid_asset = True
if not mermaid_asset:
app.config['bsk_templates'].site_scripts.append({
'href': 'https://unpkg.com/[email protected]/dist/mermaid.min.js'
})
# noinspection PyUnresolvedReferences
return render_template(f"app/views/project_details.j2", project=project, active_nav_item='/projects')
@app.route('/meta/health/canary', methods=['get', 'options'])
def meta_healthcheck_canary():
"""
Returns whether this service is healthy
This healthcheck checks the application itself (assumed to be healthy if this method can be executed).
If healthy a 204 No Content response is returned, if unhealthy a 503 Service Unavailable response is returned.
This healthcheck is binary and does not return any details to reduce payload size and prevent leaking sensitive
data.
Other healthcheck's should be used where more details are required. This healthcheck is intended for use with
load balancers to give early indication of a service not being available.
"""
return '', HTTPStatus.NO_CONTENT
return app