def create_weekly_lab_report(org_id):
"""Create a weekly lab report for lab organization owners that
provides summary statistics and figures of their historic performance
and forecasts for the coming week, measuring forecast errors
of prior forecasts.
The weekly lab report principally provides:
- Timeseries statistics and figures for all key series.
- Calculation of key performance indicators, including:
* Average turnaround time per sample.
* Number of new clients.
Args:
org_id (str): The organization ID of the company for which to prepare
a report.
"""
summary_stats = {}
# Get a company's data models.
data_models = firebase.get_collection(f'organizations/{org_id}/data_models')
# Calculate summary statistics for each data model.
for data_model in data_models:
key = data_model['key']
print(key)
summary_stats[key] = {}
# Get model data.
data = firebase.get_collection(f'organizations/{org_id}/{key}')
# Calculate model data summary statistics.
summary_stats[key]['total'] = len(data)
# TODO: Calculate daily, weekly, and monthly stats.
# TODO: Calculate key performance indicators.
# - Average turnaround time per sample.
# - Number of new clients.
# Upload the summary statistics.
update_document(f'organizations/{org_id}/, summary_stats)
# TODO: Create figures.
# TODO: Create LaTeX tables and text.
# Optional: Email the report.
return summary_stats
def labs(request):
"""Get laboratory information (public API endpoint)."""
# Get organization(s).
if request.method == 'GET':
filters = []
order_by = None
# Get a specific organization.
organization_id = request.query_params.get('organization_id')
print('Organization ID:', organization_id)
if organization_id:
filters.append({
'key': 'slug',
'operation': '==',
'value': organization_id
})
# Get all organizations in a state
state = request.query_params.get('state')
if state:
filters.append({'key': 'state', 'operation': '==', 'value': state})
order_by = 'name'
# Query and return the docs.
docs = get_collection('labs', filters=filters, order_by=order_by)
print('Returning docs:', docs)
return Response({'data': docs}, status=200)
def get_user_context(request, context):
"""Get the user-specific context.
Args:
request (HTTPRequest): A request to check for a user session.
context (dict): Existing page context.
Returns
context (dict): Page context updated with any user-specific context.
"""
print('Getting user context by verifying session.')
try:
claims = auth.verify_session(request)
except:
print('Invalid, not present, or expired session cookie.')
return context
if claims:
uid = claims['uid']
print('User verified from session cookie:', claims['email'])
query = {'key': 'team', 'operation': 'array_contains', 'value': uid}
organizations = get_collection('organizations', filters=[query])
user_data = get_document(f'users/{uid}')
context['organizations'] = organizations
context['user'] = {**claims, **user_data}
else:
context['organizations'] = []
context['user'] = {}
return context
def send_results(org_id, sample_ids, recipients, method='email'):
"""Send results to their recipients with email or text message.
Sample data is retrieved in batches of 10 to use for text or email.
Args:
org_id (str): The organization sending the results.
sample_ids (list): A list of sample IDs for which to send results.
recipients (list): A list of recipients, either a list of emails or
phone numbers
method (str): The method of result delivery, `email` or `text`,
with `email` by default.
Returns:
(list): A list of all success indicators (bool).
"""
samples = []
batches = [sample_ids[i:i + 10] for i in range(0, len(sample_ids), 10)]
for batch in batches:
filters = [{'key': 'sample_id', 'operation': 'in', 'value': batch}]
docs = get_collection(f'organizations/{org_id}/samples',
filters=filters)
samples = [*samples, *docs]
if method == 'text':
successes = text_results(samples, recipients)
else:
successes = email_results(samples, recipients)
return successes
def promotions(request):
"""Record a promotion, by getting promo code,
finding any matching promotion document,
and updating the views."""
try:
data = loads(request.body)
promo_code = data['promo_code']
matches = get_collection('promos/events/promo_stats',
filters=[
{
'key': 'hash',
'operation': '>=',
'value': promo_code
},
{
'key': 'hash',
'operation': '<=',
'value': '\uf8ff'
},
])
match = matches[0]
promo_hash = match['hash']
timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
increment_value(f'promos/events/promo_stats/{promo_hash}', 'views')
add_to_array(f'promos/events/promo_stats/{promo_hash}', 'viewed_at',
timestamp)
# Optional: If user has an account,
# record which user visited in viewed_by collection.
return JsonResponse({'message': {'success': True}})
except:
return JsonResponse({'message': {'success': False}})
def get_lab_data(self, context):
"""Get a lab's data from Firestore."""
slug = self.kwargs.get('lab')
filters = [{'key': 'slug', 'operation': '==', 'value': slug}]
labs = get_collection('labs', filters=filters)
if labs:
context['lab'] = labs[0]
else:
context['lab'] = {}
return context
def create_coas(request):
"""Generate certificates of analysis."""
# Authenticate the user.
claims, status, org_id = authorize_user(request)
if status != 200:
return Response(claims, status=status)
# Get posted samples.
posted_data = loads(request.body.decode('utf-8'))
sample_ids = posted_data['sample_ids']
# Create certificates for each sample.
data = []
for sample_id in sample_ids:
# Get the sample data.
sample_data = get_document(
f'organizations/{org_id}/samples/{sample_id}')
# Get the results for each sample. If there are no results,
# then get the measurements for each sample and calculate
# the results for each sample. Add a empty dictionary if missing everything.
sample_results = get_collection(f'organization/{org_id}/results',
order_by='updated_at',
desc=True,
filters=[{
'key': 'sample_id',
'operation': '==',
'value': sample_id
}])
if not sample_results:
sample_results = calculate_results(request)
if not sample_results:
sample_results = [{}]
# Define the certificate context.
context = {**sample_data, **sample_results[0]}
# Get the certificate template.
template_name = sample_data.get('coa_template_ref', DEFAULT_TEMPLATE)
# Create the PDF, keeping the data.
# Efficiency gain: Keep the template in /tmp so they don't have
# to be downloaded each iteration.
certificate = generate_coas(
context,
coa_template=template_name,
# output_pages=pages,
# limits=limits
)
data.append(certificate)
# Return list of certificate data.
return Response({'data': data}, status=200)
def lab_logs(request, org_id, format=None):
"""Get or create lab logs."""
if request.method == 'GET':
data = get_collection(f'labs/{org_id}/logs')
return Response({'data': data}, content_type='application/json')
elif request.method == 'POST':
# TODO: Create a log.
return Response({'data': 'Under construction'},
content_type='application/json')
def set_updated_at(ref: str):
"""Set the `updated_at` field on all documents in a collection.
Args:
ref (str): The original collection.
"""
print(f'Setting `updated_at` for all documents in {ref}...')
updated_at = datetime.now().isoformat()
docs = get_collection(ref)
for doc in docs:
entry = {'updated_at': updated_at}
update_document(ref + '/' + doc['id'], entry)
print(f'Finished setting `updated_at` for all documents in {ref}.')
def lab_analyses(request, org_id, format=None):
"""
Get or update (TODO) lab analyses.
"""
if request.method == 'GET':
data = get_collection(f'labs/{org_id}/analyses')
return Response({'data': data}, content_type='application/json')
elif request.method == 'POST':
# TODO: Create an analysis.
return Response({'data': 'Under construction'},
content_type='application/json')
def state_data(request, state=None):
"""Get or update data for a given state."""
# Optional: Allow authenticated users to edit state data?
# claims = auth.authenticate_request(request)
# if request.method == 'GET':
# if request.method == 'POST':
# uid = claims['uid']
if state:
data = get_document(f'public/data/state_data/{state}')
else:
data = get_collection('public/data/state_data', order_by='state')
response = {'success': True, 'message': '', 'data': data}
return Response(response)
def move_collection(ref, dest, delete=False):
"""Move one collection to another collection.
Args:
ref (str): The original collection.
dest (str): The new collection.
delete (bool): Wether or not to delete the original documents,
`False` by default.
"""
docs = firebase.get_collection(ref)
for doc in docs:
firebase.update_document(dest + '/' + doc['id'], doc)
if delete:
firebase.delete_document(ref + '/' + doc['id'])
def lab(request, format=None):
"""Get or update information about a lab."""
# Query labs.
if request.method == 'GET':
limit = request.query_params.get('limit', None)
order_by = request.query_params.get('order_by', 'state')
# TODO: Get any filters from dict(request.query_params)
labs = get_collection('labs',
order_by=order_by,
limit=limit,
filters=[])
return Response({'data': labs}, content_type='application/json')
def get_api_key_hmacs(request, *args, **argv): #pylint: disable=unused-argument
"""Get a user's API key HMAC information.
Args:
request (HTTPRequest): A request to get the user's HMAC information.
Returns:
(JsonResponse): A JSON response containing the API key HMAC
information in a `data` field.
"""
user_claims = verify_session(request)
uid = user_claims['uid']
query = {'key': 'uid', 'operation': '==', 'value': uid}
docs = get_collection('admin/api/api_key_hmacs', filters=[query])
return JsonResponse({'status': 'success', 'data': docs})
def set_updated_at(ref):
"""Set the `updated_at` field on all documents in a collection.
Args:
ref (str): The original collection.
dest (str): The new collection.
delete (bool): Wether or not to delete the original documents,
`False` by default.
"""
updated_at = datetime.now().isoformat()
docs = firebase.get_collection(ref)
for doc in docs:
entry = {'updated_at': updated_at}
firebase.update_document(ref + '/' + doc['id'], entry)
def download_dataset(claims, collection, data_points):
"""Download a given dataset."""
# Get the user's data, returning if not authenticated.
try:
uid = claims['uid']
user_email = claims['email']
name = claims.get('name', 'Unknown')
except KeyError:
return None, None
# Get data points in specified order.
collection_data = get_collection(collection, order_by='state')
dataframe = DataFrame.from_dict(collection_data, orient='columns')
data = dataframe[data_points]
# Convert JSON to CSV.
with NamedTemporaryFile(delete=False) as temp:
temp_name = temp.name + '.csv'
data.to_csv(temp_name, index=False)
temp.close()
# Post a copy of the data to Firebase storage.
now = datetime.now()
timestamp = now.strftime('%Y-%m-%d_%H-%M-%S')
destination = 'public/data/downloads/'
data_type = collection.split('/')[-1]
filename = f'{data_type}_{timestamp}.csv'
ref = destination + filename
upload_file(ref, temp_name, bucket_name=STORAGE_BUCKET)
# Create an activity log.
log_entry = {
'data_points': len(data),
'file': ref,
'email': user_email,
'name': name,
'uid': uid,
}
create_log(
ref='logs/website/downloads',
claims=claims,
action=f'User ({user_email}) downloaded {data_type} data.',
log_type='download',
key=f'download_{data_type}_data',
changes=log_entry,
)
# Return the file that can be downloaded.
return temp_name, filename
def move_collection(ref: str, dest: str, delete: Optional[bool] = False):
"""Move one collection to another collection.
Args:
ref (str): The original collection.
dest (str): The new collection.
delete (bool): Wether or not to delete the original documents,
`False` by default.
"""
print(f'Moving documents from {ref} to {dest}...')
docs = get_collection(ref)
for doc in docs:
update_document(dest + '/' + doc['id'], doc)
if delete:
delete_document(ref + '/' + doc['id'])
print(f'Moved all documents in {ref} to {dest}.')
def lab_data(request, license_number=None):
"""Get laboratory information (public API endpoint)."""
data = []
if request.method == 'GET':
# Get a specific organization.
organization_id = request.query_params.get('organization_id')
if organization_id and organization_id != 'undefined':
data = get_document(f'public/data/labs/{organization_id}')
else:
# Define query parameters.
filters = []
order_by = request.query_params.get('order_by', 'name')
limit = request.query_params.get('limit')
state = request.query_params.get('state')
# Optional: Implement more queries the user can use.
# - name
# - analyses?
# Apply user-specified filters.
if license_number:
filters.append({
'key': 'license',
'operation': '==',
'value': license_number
})
elif state:
filters.append({
'key': 'state',
'operation': '==',
'value': state
})
# Query and return the docs.
data = get_collection(
'public/data/labs',
desc=False,
filters=filters,
limit=limit,
order_by=order_by,
)
# Return data in a response.
response = {'success': True, 'data': data}
return Response(response, status=200)
def delete_user_pin(request, *args, **argv): #pylint: disable=unused-argument
"""Delete all pins for a given user, removing the data stored with their hash.
Args:
request (HTTPRequest): A request to get the user's session.
Returns:
(JsonResponse): A JSON response containing the API key in an
`api_key` field.
"""
user_claims = authenticate_request(request)
uid = user_claims['uid']
query = {'key': 'uid', 'operation': '==', 'value': uid}
existing_pins = get_collection('admin/api/pin_hmacs', filters=[query])
for pin in existing_pins:
code = pin['hmac']
delete_document(f'admin/api/pin_hmacs/{code}')
delete_field(f'users/{uid}', 'pin_created_at')
create_log(f'users/{uid}/logs', user_claims, 'Deleted pin.', 'pin', 'pin_delete', [{'deleted_at': datetime.now().isoformat()}])
return JsonResponse({'success': True, 'message': 'User pin deleted.'})
def get_user_data(request: Any, context: dict) -> dict:
"""Get user-specific context.
Args:
request (HTTPRequest): A request to check for a user session.
context (dict): Existing page context.
Returns
context (dict): Page context updated with any user-specific context.
"""
claims = authenticate_request(request)
try:
uid = claims['uid']
query = {'key': 'team', 'operation': 'array_contains', 'value': uid}
organizations = get_collection('organizations', filters=[query])
user_data = get_document(f'users/{uid}')
context['organizations'] = organizations
context['user'] = {**claims, **user_data}
except KeyError:
context['organizations'] = []
context['user'] = {}
return context
def get_page_data(context: dict) -> dict:
"""Get all data for a page from Firestore.
Args:
context (dict): A dictionary of existing page context.
Returns
(dict): The context updated with any page-specific data.
"""
namespaces = []
try:
namespace = context['page']
namespaces.append(page_data[namespace])
except KeyError:
pass
try:
namespace = context['section']
namespaces.append(page_data[namespace])
except KeyError:
pass
for namespace in namespaces:
try:
documents = namespace['documents']
for item in documents:
context[item['name']] = get_document(item['ref'])
except KeyError:
pass
try:
collections = namespace['collections']
for item in collections:
context[item['name']] = get_collection(
item['ref'],
limit=item.get('limit'),
order_by=item.get('order_by'),
desc=item.get('desc'),
filters=item.get('filters'),
)
except KeyError:
pass
return context
def get_data(
ref: str,
datafile: Optional[str] = '',
order_by: Optional[str] = '',
) -> List[dict]:
"""Get a dataset saved in Firestore.
Args:
ref (str): The reference to a collection.
datafile (str): Save the data locally to the project's `.datasets`
if a data file is given.
order_by (str): An optional field to order the results by.
Returns:
(list): Returns a list of data (dict).
"""
print('Getting data...')
database = initialize_firebase()
data = get_collection(ref, database=database, order_by=order_by)
print('Found {} observations.'.format(len(data)))
if datafile:
print('Saving data...')
save_data(data, datafile)
print('Saved data to', datafile)
return data
def analysis_data(request, analysis_id=None):
"""Get data about lab analyses (public API endpoint)."""
data = []
collection = 'public/data/analyses'
if request.method == 'GET':
# Get a specific observation.
if analysis_id is not None:
data = get_document(f'{collection}/{analysis_id}')
# Otherwise query observations.
else:
# Define query parameters.
filters = []
order_by = request.query_params.get('order_by', 'name')
limit = request.query_params.get('limit')
# Optional: Implement more queries the user can use.
# Apply user-specified filters.
# param = request.query_params.get('param')
# if param:
# filters.append({'key': 'param', 'operation': '==', 'value': param})
# Query and return the docs.
data = get_collection(
collection,
desc=False,
filters=filters,
limit=limit,
order_by=order_by,
)
# Return the data.
response = {'success': True, 'data': data}
return Response(response, status=200)
def join_organization(request):
"""Send the owner of an organization a request for a user to join."""
# Identify the user.
claims = authenticate_request(request)
uid = claims['uid']
user_email = claims['email']
post_data = loads(request.body.decode('utf-8'))
organization = post_data.get('organization')
# Return an error if the organization doesn't exist.
query = {'key': 'organization', 'operation': '==', 'value': organization}
organizations = get_collection('organizations', filters=[query])
if not organizations:
message = 'Organization does not exist. Please check the organization name and try again.'
return Response({'success': False, 'message': message}, status=400)
# Send the owner an email requesting to add the user to the organization's team.
org_email = organizations[0]['email']
text = f"A user with the email address {user_email} would like to join your organization, \
{organization}. Do you want to add this user to your organization's team? Please \
reply YES or NO to confirm."
paragraphs = []
# TODO: Generate confirm, decline, and unsubscribe links with HMACs from user's uid and owner's uid.
user_hmac = ''
owner_hmac = ''
# Optional: Find new home's for endpoints in api and cannlytics_website
confirm_link = f'https://console.cannlytics.com/api/organizations/confirm?hash={owner_hmac}&member={user_hmac}'
decline_link = f'https://console.cannlytics.com/api/organizations/decline?hash={owner_hmac}&member={user_hmac}'
unsubscribe_link = f'https://console.cannlytics.com/api/unsubscribe?hash={owner_hmac}'
# html_message = render_to_string('templates/console/emails/action_email_template.html', {
# 'recipient': org_email,
# 'paragraphs': paragraphs,
# 'primary_action': 'Confirm',
# 'primary_link': confirm_link,
# 'secondary_action': 'Decline',
# 'secondary_link': decline_link,
# 'unsubscribe_link': unsubscribe_link,
# })
# TODO: Skip sending email if owner is unsubscribed.
# send_mail(
# subject="Request to join your organization's team.",
# message=text,
# from_email=DEFAULT_FROM_EMAIL,
# recipient_list=LIST_OF_EMAIL_RECIPIENTS,
# fail_silently=False,
# html_message=html_message
# )
# Create activity logs.
# create_log(f'users/{uid}/logs', claims, 'Requested to join an organization.', 'users', 'user_data', [post_data])
# create_log(f'organization/{uid}/logs', claims, 'Request from a user to join the organization.', 'organizations', 'organization_data', [post_data])
message = f'Request to join {organization} sent to the owner.'
return Response({
'success': True,
'message': message
},
content_type='application/json')
def get_youtube_video_views(event, context):
"""Get video views for YouTube channel, saving them to
their corresponding video data in Firestore.
Triggered from a message on a Cloud Pub/Sub topic.
Args:
event (dict): Event payload.
context (google.cloud.functions.Context): Metadata for the event.
"""
pubsub_message = base64.b64decode(event['data']).decode('utf-8')
if pubsub_message != 'success':
return
# Get YouTube API key saved as an environment variable.
print('Getting API Key...')
api_key = os.environ.get('YOUTUBE_API_KEY')
if api_key is not None:
print('Found API key')
# if api_key is None:
# import yaml
# dir_path = os.path.dirname(os.path.realpath(__file__))
# with open(f'{dir_path}/env.yaml', 'r') as env:
# config = yaml.load(env, Loader=yaml.FullLoader)
# api_key = config['YOUTUBE_API_KEY']
# credentials = config['GOOGLE_APPLICATION_CREDENTIALS']
# os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials
# Initialize Firebase.
# initialize_firebase()
print('Initializing Firebase')
try:
initialize_app()
except ValueError:
pass
database = firestore.client()
print('Initialized Firebase')
# Get all videos.
print('Getting all videos')
videos = get_collection(
'public/videos/video_data',
order_by='published_at',
desc=True,
database=database,
)
print('Found %i videos' % len(videos))
# Get view and like count for each video.
refs = []
updates = []
print('Getting views and likes...')
for video in videos:
video_id = video['id']
youtube_id = video['youtube_id']
url = f'{BASE}/videos?part=statistics&id={youtube_id}&key={api_key}'
response = requests.get(url)
data = response.json()
stats = data['items'][0]['statistics']
updates.append({
'views': stats['viewCount'],
'likes': stats['likeCount'],
})
refs.append(f'public/videos/video_data/{video_id}')
# Update all videos with the latest view and like count.
print('Found all views and likes. Preparing to update videos...')
update_documents(refs, updates, database=database)
print('Updated all %i videos with view counts.' % len(videos))
def test_firestore():
"""Test Firestore functions by managing a test document."""
# Initialize Firebase.
env = environ.Env()
env.read_env('../.env')
credentials = env('GOOGLE_APPLICATION_CREDENTIALS')
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials
db = firebase.initialize_firebase()
# Create a collection reference.
col_ref = firebase.create_reference(db, 'tests')
assert isinstance(col_ref, CollectionReference) == True
# Create a document reference.
doc_ref = firebase.create_reference(db, 'tests/firebase_test')
assert isinstance(doc_ref, DocumentReference) == True
# Create a document.
firebase.update_document('tests/firebase_test', {'user': '******'})
# Update a document.
firebase.update_document('tests/firebase_test', {'test': 'firebase_test'})
# Get the document.
data = firebase.get_document('tests/firebase_test')
assert data['user'] == 'CannBot'
assert data['test'] == 'firebase_test'
# Get a collection.
filters = [{'key': 'test', 'operation': '==', 'value': 'firebase_test'}]
docs = firebase.get_collection('tests', filters=filters)
assert docs[0]['test'] == 'firebase_test'
# Add an element to an array in a document.
firebase.add_to_array('tests/firebase_test', 'likes', 'Testing')
data = firebase.get_document('tests/firebase_test')
assert 'Testing' in data['likes']
# Remove an element from an array in a document.
firebase.remove_from_array('tests/firebase_test', 'likes', 'Testing')
data = firebase.get_document('tests/firebase_test')
assert 'Testing' not in data['likes']
# Increment a value in a document.
firebase.increment_value('tests/firebase_test', 'runs')
data = firebase.get_document('tests/firebase_test')
assert data['runs'] > 0
# Import .csv data to Firestore.
ref = 'tests/test_collections/licensees'
data_file = './assets/data/licensees_partial.csv'
firebase.import_data(db, ref, data_file)
# TODO: Test import .xlsx data to Firestore.
# TODO: Test import .txt data to Firestore.
# Export data to .csv from Firestore.
output_csv_file = './assets/data/licensees_test.csv'
output_xlsx_file = './assets/data/licensees_test.xlsx'
firebase.export_data(db, ref, output_csv_file)
# Export data to .xlsx from Firestore.
firebase.export_data(db, ref, output_xlsx_file)
import os
import environ
# Initialize Firebase.
env = environ.Env()
env.read_env('../../.env')
credentials = env('GOOGLE_APPLICATION_CREDENTIALS')
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials
db = initialize_firebase()
# Get parameters.
org_id = 'test-company'
# For each data model, calculate stats
ref = f'organizations/{org_id}/data_models'
data_models = get_collection(ref)
for data_model in data_models[:1]:
# Calculate stats for given data model.
key = data_model['key']
data_ref = f'organizations/{org_id}/{key}'
data = get_collection(data_ref)
# TODO: Daily stats
# TODO: Weekly stats
# TODO: Monthly stats
# TODO: Summary stats (total, mean, max, min, std. dev)
def labs(request, format=None):
"""Get or update information about labs."""
# Query labs.
if request.method == 'GET':
limit = request.query_params.get('limit', None)
order_by = request.query_params.get('order_by', 'state')
# TODO: Get any filters from dict(request.query_params)
labs = get_collection('labs',
order_by=order_by,
limit=limit,
filters=[])
return Response({'data': labs}, content_type='application/json')
# Update a lab given a valid Firebase token.
elif request.method == 'POST':
# Check token.
try:
claims = auth.authenticate(request)
except:
return Response({'error': 'Could not auth.authenticate.'},
status=status.HTTP_400_BAD_REQUEST)
# Get the posted lab data.
lab = request.data
org_id = lab['id']
lab['slug'] = slugify(lab['name'])
# TODO: Handle adding labs.
# Create uuid, latitude, and longitude, other fields?
# Determine any changes.
existing_data = get_document(f'labs/{org_id}')
changes = []
for key, after in lab:
before = existing_data[key]
if before != after:
changes.append({'key': key, 'before': before, 'after': after})
# Get a timestamp.
timestamp = datetime.now().isoformat()
lab['updated_at'] = timestamp
# Create a change log.
log_entry = {
'action': 'Updated lab data.',
'type': 'change',
'created_at': lab['updated_at'],
'user': claims['uid'],
'user_name': claims['display_name'],
'user_email': claims['email'],
'photo_url': claims['photo_url'],
'changes': changes,
}
update_document(f'labs/{org_id}/logs/{timestamp}', log_entry)
# Update the lab.
update_document(f'labs/{org_id}', lab)
return Response(log_entry, status=status.HTTP_201_CREATED)
def automatic_collection(org_id=None, env_file='.env', minutes_ago=None):
"""Automatically collect results from scientific instruments.
Args:
org_id (str): The organization ID to associate with instrument results.
env_file (str): The environment variable file, `.env` by default.
Either a `GOOGLE_APPLICATION_CREDENTIALS` or a
`CANNLYTICS_API_KEY` is needed to run the routine.
minutes_ago (int): The number of minutes in the past to restrict
recently modified files.
Returns:
(list): A list of measurements (dict) that were collected.
"""
# Try to initialize Firebase, otherwise an API key will be used.
try:
env = environ.Env()
env.read_env(env_file)
credentials = env('GOOGLE_APPLICATION_CREDENTIALS')
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials
initialize_firebase()
except:
pass
# Get the organization ID from the .env file if not specified.
if not org_id:
org_id = env('CANNLYTICS_ORGANIZATION_ID')
# Format the last modified time cut-off as a datetime.
last_modified_at = None
if minutes_ago:
last_modified_at = datetime.now() - timedelta(minutes=minutes_ago)
# Get the instruments, trying Firestore, then the API.
try:
ref = f'organizations/{org_id}/instruments'
instrument_data = get_collection(ref)
except:
api_key = env('CANNLYTICS_API_KEY')
headers = {
'Authorization': 'Bearer %s' % api_key,
'Content-type': 'application/json',
}
url = f'{API_BASE}/instruments?organization_id={org_id}'
response = requests.get(url, headers=headers)
instrument_data = response.json()['data']
# Iterate over instruments, collecting measurements.
measurements = []
for instrument in instrument_data:
# Iterate over analyses that the instrument may be running.
try:
analyses = instrument.get('analyses', '').split(',')
except AttributeError:
continue # FIXME: Handle missing analyses more elegantly.
analyses = [x.strip() for x in analyses]
for n in range(len(analyses)):
# Optional: Handle multiple data paths more elegantly.
analysis = analyses[n]
try:
data_paths = instrument['data_path'].split(',')
except AttributeError:
continue # No data path.
data_paths = [x.strip() for x in data_paths]
data_path = data_paths[n]
if not data_path:
continue
# Identify the analysis being run and identify the import routine.
# Optional: Identify more elegantly.
if 'micro' in analysis or 'MICR' in analysis:
import_method = globals()['import_micro']
elif 'metal' in analysis or 'HEAV' in analysis:
import_method = globals()['import_heavy_metals']
else:
import_method = globals()['import_results']
# Search for recently modified files in the instrument directory
# and parse any recently modified file.
for root, _, filenames in os.walk(data_path):
for filename in filenames:
if filename.endswith('.xlsx') or filename.endswith('.xls'):
data_file = os.path.join(root, filename)
modifed_at = os.stat(data_file).st_mtime
# FIXME: Ensure date restriction works.
if last_modified_at:
if modifed_at < last_modified_at:
continue
samples = import_method(data_file)
if isinstance(samples, dict):
sample_data = {**instrument, **samples}
measurements.append(sample_data)
else:
for sample in samples:
sample_data = {**instrument, **sample}
measurements.append(sample_data)
# Upload measurement data to Firestore.
now = datetime.now()
updated_at = now.isoformat()
for measurement in measurements:
try:
measurement['sample_id'] = measurement['sample_name']
except:
continue # Already has `sample_id`.
# TODO: Format a better measurement ID.
measurement_id = measurement.get(
'acq_inj_time') # E.g. 12-Jun-21, 15:21:07
if not measurement_id:
# measurement_id = now.strftime('%d-%b-%y-%H-%M-%S') + '_' + str(measurement['sample_id'])
measurement_id = measurement['sample_id']
else:
try:
measurement_id = measurement_id.replace(',', '').replace(
' ', '-').replace(':', '-')
except AttributeError:
pass
measurement_id = str(measurement_id) + '_' + str(
measurement['sample_id'])
measurement['measurement_id'] = measurement_id
measurement['updated_at'] = updated_at
ref = f'organizations/{org_id}/measurements/{measurement_id}'
try:
update_document(ref, measurement)
except:
url = f'{API_BASE}/measurements/{measurement_id}?organization_id={org_id}'
response = requests.post(url, json=measurement, headers=headers)
print('Uploaded measurement:', ref)
# Upload result data to Firestore
for result in measurement['results']:
analyte = result['analyte']
result_id = f'{measurement_id}_{analyte}'
result['sample_id'] = measurement['sample_name']
result['result_id'] = result_id
result['measurement_id'] = measurement_id
result['updated_at'] = updated_at
ref = f'organizations/{org_id}/results/{result_id}'
try:
update_document(ref, result)
except:
url = f'{API_BASE}/results/{result_id}?organization_id={org_id}'
response = requests.post(url, json=result, headers=headers)
print('Uploaded result:', ref)
# Return the measurements
return measurements
def organizations(request, organization_id=None, type='lab'):
"""Get, create, or update organizations.
E.g.
```
organization = {
'owner': [],
'name': '',
'license': '',
'type': '',
'team': [],
'support': '',
}
```
"""
# Get endpoint variables.
model_type = 'organizations'
_, project_id = google.auth.default()
claims = authenticate_request(request)
uid = claims['uid']
print('User request to organizations:', uid)
# Get organization(s).
if request.method == 'GET':
# Get organization_id parameter
if organization_id:
print('Query organizations by ID:', organization_id)
data = get_document(f'{model_type}/{organization_id}')
print('Found data:', data)
if not data:
message = 'No organization exists with the given ID.'
return Response({
'error': True,
'message': message
},
status=404)
elif data['public']:
return Response({'data': data}, status=200)
elif uid not in data['team']:
message = 'This is a private organization and you are not a team member. Request to join before continuing.'
return Response({
'error': True,
'message': message
},
status=400)
else:
return Response({'data': data}, status=200)
# TODO: Get query parameters.
keyword = request.query_params.get('name')
if keyword:
print('Query by name:', keyword)
query = {'key': 'name', 'operation': '==', 'value': keyword}
docs = get_collection(model_type, filters=[query])
return Response({'data': docs}, status=200)
# Get all of a user's organizations
else:
query = {
'key': 'team',
'operation': 'array_contains',
'value': uid
}
docs = get_collection(model_type, filters=[query])
return Response({'data': docs}, status=200)
# Optional: Get list of other organizations.
# Check if user is in organization's team, otherwise,
# only return publically available information.
# Optional: Try to get facility data from Metrc.
# facilities = track.get_facilities()
# Create or update an organization.
elif request.method == 'POST':
# Update an organization with the posted data if there is an ID.
data = loads(request.body.decode('utf-8'))
if organization_id:
# Return an error if the organization already exists
# and the user is not part of the organization's team.
doc = get_document(f'{model_type}/{organization_id}')
if not doc:
message = 'No data exists for the given ID.'
return Response({
'error': True,
'message': message
},
status=400)
organization_id = doc['uid']
team_list = claims.get('team', [])
owner_list = claims.get('owner', [])
if uid not in team_list and organization_id not in owner_list:
message = 'You do not currently belong to this organization. Request to join before continuing.'
return Response({
'error': True,
'message': message
},
status=400)
# If an organization already exists, then only the owner
# can edit the organization's team.
if uid != doc['owner']:
data['team'] = doc['team']
# Store posted API keys as secrets.
# FIXME: Update licenses if they are being edited.
new_licenses = data.get('licenses')
if new_licenses:
licenses = doc.get('licenses', [])
for license_data in new_licenses:
license_number = license_data['license_number']
secret_id = f'{license_number}_secret'
try:
create_secret(project_id, secret_id,
license_data['user_api_key'])
except:
pass
secret = add_secret_version(project_id, secret_id,
license_data['user_api_key'])
version_id = secret.split('/')[-1]
license_data['user_api_key_secret'] = {
'project_id': project_id,
'secret_id': secret_id,
'version_id': version_id,
}
del license_data['user_api_key']
licenses.append(license_data)
doc['licenses'] = licenses
# Create organization if it doesn't exist
# All organizations have a unique `organization_id`.
else:
doc = {}
organization_id = slugify(data['name'])
doc['organization_id'] = organization_id
doc['team'] = [uid]
doc['owner'] = uid
# Identify created organization type.
doc['type'] = type
# All organizations start with the standard data models.
# FIXME: Remove data models that have permissions
# if the user does not have sufficient claims.
data_models = get_collection('public/state/data_models')
for data_model in data_models:
key = data_model['key']
update_document(
f'{model_type}/{organization_id}/data_models/{key}',
data_model)
# Create or update the organization in Firestore.
entry = {**doc, **data}
print('Entry:', entry)
update_document(f'{model_type}/{organization_id}', entry)
# FIXME:
# On organization creation, the creating user get custom claims.
update_custom_claims(uid,
claims={
'owner': organization_id,
'team': organization_id
})
# TODO: Owners can add other users to the team and
# the receiving user then gets the claims.
# team: [organization_id, ...]
# Create activity log.
changes = [data]
create_log(f'{model_type}/{uid}/logs',
claims=claims,
action='Updated organization data.',
log_type=model_type,
key=f'{model_type}_data',
changes=changes)
return Response({
'data': entry,
'success': True
},
content_type='application/json')
elif request.method == 'DELETE':
# TODO: Only user's with organization_id in owner claim can delete the organization.
return Response({'error': 'not_implemented'},
content_type='application/json')
