def test_publish_data(dataset):
"""Publish a dataset on the data market."""
# Initialize Ocean market.
ocean, config = initialize_ocean_market()
# Mint a test OCEAN.
os.environ['FACTORY_DEPLOYER_PRIVATE_KEY'] = config['FACTORY_DEPLOYER_PRIVATE_KEY']
mint_fake_OCEAN(ocean.config)
# Publish a dataset.
data_token, asset = market.publish_data(
ocean,
config.get(SELLER_KEY),
files=dataset['files'],
name=dataset['datatoken_name'],
symbol=dataset['datatoken_symbol'],
author=dataset['author'],
data_license=dataset['license'],
)
# Upload the datatoken and asset information.
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = config['GOOGLE_APPLICATION_CREDENTIALS']
initialize_firebase()
ref = f'public/market/datasets/{asset.asset_id}'
entry = {**dataset, **asset.as_dictionary()}
update_document(ref, entry)
return data_token, asset
def upload_contributors(org_name):
"""Get Github contributors and save them to Firestore.
Args:
org_name (str): The name of a GitHub organization.
Returns:
(list): A list of users (dict).
"""
users = []
client = Github()
org = client.get_organization(org_name)
repos = org.get_repos()
initialize_firebase()
for repo in repos:
contributors = repo.get_contributors()
for user in contributors:
if user.name not in users:
users.append(user.name)
data = {
'company': user.company,
'description': user.bio,
'name': user.name,
'location': user.location,
'image': user.avatar_url,
'url': user.html_url,
'slug': user.login,
}
update_document(f'public/contributors/contributor_data/{user.id}', data)
return users
def get_context_data(self, **kwargs):
"""Get context that is used on all pages. The context is retrieved
dynamically from the app's state. The user's permissions are verified
on every request. User-specific context and data is returned depending
on the page. Information about data models is provided to all pages."""
context = super(BaseMixin, self).get_context_data(**kwargs)
context = get_page_context(self.kwargs, context)
initialize_firebase()
context = get_user_data(self.request, context)
context = get_page_data(context)
save_analytics(self.request, context)
return context
def test_storage():
"""Test Firebase Storage by managing a test file."""
# Get the path to your service account.
from dotenv import dotenv_values
config = dotenv_values('./env')
key_path = config['GOOGLE_APPLICATION_CREDENTIALS']
bucket_name = config['FIREBASE_STORAGE_BUCKET']
# Initialize Firebase.
firebase.initialize_firebase(key_path, bucket_name)
# Define file names.
bucket_folder = 'tests/assets/pdfs'
destination_blob_name = 'tests/assets/pdfs/pandas_cheat_sheet.pdf'
local_folder = './assets/pdfs'
source_file_name = './assets/pdfs/Pandas_Cheat_Sheet.pdf'
download_folder = './assets/downloads/pdfs'
download_file_name = './assets/downloads/pdfs/Pandas_Cheat_Sheet.pdf'
file_name = 'pandas_cheat_sheet.pdf'
file_copy = 'pandas_cheat_sheet_copy.pdf'
newfile_name = 'tests/assets/pdfs/' + file_copy
# Upload a file to a Firebase Storage bucket, with and without bucket name.
firebase.upload_file(destination_blob_name, source_file_name)
firebase.upload_file(destination_blob_name, source_file_name, bucket_name)
# Upload all files in a folder to a Firebase Storage bucket, with and without bucket name.
firebase.upload_files(bucket_folder, local_folder)
firebase.upload_files(bucket_folder, local_folder, bucket_name)
# List all files in the Firebase Storage bucket folder, with and without bucket name.
files = firebase.list_files(bucket_folder)
assert isinstance(files, list)
files = firebase.list_files(bucket_folder, bucket_name)
assert isinstance(files, list)
# Download a file from Firebase Storage, with and without bucket name.
firebase.download_file(destination_blob_name, download_file_name)
firebase.download_file(destination_blob_name, download_file_name, bucket_name)
# Download all files in a given Firebase Storage folder, with and without bucket name.
firebase.download_files(bucket_folder, download_folder)
firebase.download_files(bucket_folder, download_folder, bucket_name)
# Rename a file in the Firebase Storage bucket, with and without bucket name.
firebase.rename_file(bucket_folder, file_name, newfile_name)
firebase.rename_file(bucket_folder, file_name, newfile_name, bucket_name)
# Delete a file from the Firebase Storage bucket, with and without bucket name.
firebase.delete_file(bucket_folder, file_copy)
firebase.delete_file(bucket_folder, file_copy, bucket_name)
def login(request, *args, **argv): #pylint: disable=unused-argument
"""Functional view to create a user session.
Optional: Ensure that the request succeeds on the client!
"""
try:
print('Logging user in...')
authorization = request.headers.get('Authorization', '')
token = authorization.split(' ').pop()
if not token:
# return HttpResponse(status=401)
message = 'Authorization token not provided in the request header.'
return JsonResponse({
'error': True,
'message': message
},
status=401)
initialize_firebase()
print('Initialized Firebase.')
# Set session cookie in a cookie in the response.
# response = HttpResponse(status=200)
response = JsonResponse({'success': True}, status=200)
# Optional: Let user specify cookie duration?
# expires_in = timedelta(days=5)
# expires = datetime.now() + expires_in
session_cookie = create_session_cookie(token)
response['Set-Cookie'] = f'__session={session_cookie}; Path=/'
response[
'Cache-Control'] = 'public, max-age=300, s-maxage=900' # TODO: Set the expiration time
# Save session cookie in the session.
# Preferred over cookies (but cookies are still needed for production).
request.session['__session'] = session_cookie
# Verify the user, create a log, update the user as signed-in,
# and return a response with the session cookie.
claims = verify_token(token)
uid = claims['uid']
print('Verified user with Firebase Authentication:', claims['email'])
create_log(ref=f'users/{uid}/logs',
claims=claims,
action='Signed in.',
log_type='auth',
key='login')
update_document(f'users/{uid}', {'signed_in': True})
print('Logged user sign-in in Firestore:', uid)
return response
except:
# return HttpResponse(status=401)
message = 'Authorization failed in entirety. Please contact support.'
return JsonResponse({'error': True, 'message': message}, status=401)
def upload_data(
file_name: str,
collection: str,
id_key: Optional[str] = 'id',
stats_doc: Optional[str] = '',
):
""" Upload a dataset to Firestore.
Args:
datafile (str): The path to a .json file containing the data.
collection (str): The path of the collection where data will be stored.
id_key (str): The key of the ID.
stats_doc (str): An optional document to store statistics about the data.
Returns:
data (list): A list of partner data (dict).
"""
database = initialize_firebase()
with open(file_name) as datafile:
data = json.load(datafile)
print('Uploading dataset...')
for item in data:
item['updated_at'] = datetime.now().isoformat()
doc_id = item[id_key]
ref = f'{collection}/{doc_id}'
update_document(ref, item, database=database)
print('Updated:', ref)
if stats_doc:
update_document(stats_doc, {'total': len(data)}, database=database)
print('Updated:', stats_doc)
print('Finished uploading data.')
return data
def login(request):
"""Functional view to create a user session."""
try:
# Ensure that the user passed an authorization bearer token.
authorization = request.headers.get('Authorization')
token = authorization.split(' ').pop()
if not token:
message = 'Authorization token not provided in the request header.'
return JsonResponse({
'success': False,
'message': message
},
status=401)
# Initialize Firebase and verify the Firebase ID token.
initialize_firebase()
claims = verify_token(token)
uid = claims['uid']
# Create and set a session cookie in the response.
cache = f'public, max-age={SESSION_COOKIE_AGE}, s-maxage={SESSION_COOKIE_AGE}'
session_cookie = create_session_cookie(token)
response = JsonResponse({'success': True}, status=200)
response['Cache-Control'] = cache
response['Set-Cookie'] = f'__session={session_cookie}; Path=/'
# Also save the session cookie in the session.
# Note: The session is preferred over cookies,
# but cookies are currently needed for production.
request.session['__session'] = session_cookie
# Log the login and update the user as signed-in.
update_document(f'users/{uid}', {'signed_in': True})
create_log(ref=f'users/{uid}/logs',
claims=claims,
action='Signed in.',
log_type='auth',
key='login')
return response
except:
message = f'Authorization failed in entirety. Please contact {DEFAULT_FROM_EMAIL}'
return JsonResponse({'success': False, 'message': message}, status=401)
def test_storage():
"""Test Firebase Storage by managing a test file."""
# Initialize Firebase.
env = environ.Env()
env.read_env('../.env')
credentials = env('GOOGLE_APPLICATION_CREDENTIALS')
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials
firebase.initialize_firebase()
# Define file names.
bucket_name = 'cannlytics.appspot.com'
bucket_folder = 'tests/assets/pdfs'
destination_blob_name = 'tests/assets/pdfs/pandas_cheat_sheet.pdf'
local_folder = './assets/pdfs'
source_file_name = './assets/pdfs/Pandas_Cheat_Sheet.pdf'
download_folder = './assets/downloads/pdfs'
download_file_name = './assets/downloads/pdfs/Pandas_Cheat_Sheet.pdf'
file_name = 'pandas_cheat_sheet.pdf'
file_copy = 'pandas_cheat_sheet_copy.pdf'
newfile_name = 'tests/assets/pdfs/' + file_copy
# Upload a file to a Firebase Storage bucket.
firebase.upload_file(bucket_name, destination_blob_name, source_file_name)
# Upload all files in a folder to a Firebase Storage bucket.
firebase.upload_files(bucket_name, bucket_folder, local_folder)
# List all files in the Firebase Storage bucket folder.
files = firebase.list_files(bucket_name, bucket_folder)
assert isinstance(files, list) == True
# Download a file from Firebase Storage.
firebase.download_file(bucket_name, destination_blob_name, download_file_name)
# Download all files in a given Firebase Storage folder.
firebase.download_files(bucket_name, bucket_folder, download_folder)
# Rename a file in the Firebase Storage bucket.
firebase.rename_file(bucket_name, bucket_folder, file_name, newfile_name)
# Delete a file from the Firebase Storage bucket.
firebase.delete_file(bucket_name, bucket_folder, file_copy)
def test_auth():
"""Test Firebase Authentication by managing a user."""
# Initialize Firebase
env = environ.Env()
env.read_env('../.env')
credentials = env('GOOGLE_APPLICATION_CREDENTIALS')
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials
firebase.initialize_firebase()
# Create account.
name = 'CannBot'
email = '*****@*****.**'
claims = {'organizations': ['Cannlytics']}
user, _ = firebase.create_account(name, email, notification=True)
# Create and get custom claims.
firebase.create_custom_claims(user.uid, email=email, claims=claims)
custom_claims = firebase.get_custom_claims(email)
# Create custom token.
token = firebase.create_custom_token(user.uid, email=None, claims=custom_claims)
assert isinstance(token, bytes) == True
# Get user.
user = firebase.get_user(email)
# Get all users.
all_users = firebase.get_users()
assert isinstance(all_users, list) == True
# Update user.
photo_url = f'https://robohash.org/{email}?set=set5'
user = firebase.update_user(user, {'photo_url': photo_url})
assert user.photo_url == 'https://robohash.org/[email protected]?set=set5'
# Delete a user.
firebase.delete_user(user.uid)
def test_secret_manager():
"""Test Secret Manager by creating a secret, updating the version
of the secret, then deleting the secret."""
# Initialize Firebase
env = environ.Env()
env.read_env('../../.env')
credentials = env('GOOGLE_APPLICATION_CREDENTIALS')
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials
firebase.initialize_firebase()
# Mock license.
license_data = {
'license_number': 'test',
'license_type': 'lab',
'state': 'OK',
'user_api_key': '123',
}
license_number = license_data['license_number']
user_api_key = license_data['user_api_key']
# Create a secret.
project_id = 'cannlytics'
secret_id = f'{license_number}-secret'
try:
version_id = firebase.create_secret(project_id, secret_id, user_api_key)
except: # AlreadyExists error
pass # Secret may already be created.
# Add the secret's secret data.
secret = firebase.add_secret_version(project_id, secret_id, user_api_key)
version_id = secret.split('/')[-1]
# Get the secret.
# In production the secret ID and version ID are stored with the license data.
user_api_key = firebase.access_secret_version(project_id, secret_id, version_id)
assert user_api_key == license_data['user_api_key']
def test_create_log():
"""Test create a log."""
# Initialize Firebase.
env = environ.Env()
env.read_env('../.env')
credentials = env('GOOGLE_APPLICATION_CREDENTIALS')
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials
db = firebase.initialize_firebase()
# Create a test log.
user = {
'uid': 'test',
'display_name': 'CannBot',
'email': '*****@*****.**',
'photo_url': 'https://robohash.org/[email protected]',
}
logs = 'tests/test_collections/logs'
firebase.create_log(logs, user, 'Test log.', 'test', 'create_log')
def get_data(
ref: str,
datafile: Optional[str] = '',
order_by: Optional[str] = '',
) -> List[dict]:
"""Get a dataset saved in Firestore.
Args:
ref (str): The reference to a collection.
datafile (str): Save the data locally to the project's `.datasets`
if a data file is given.
order_by (str): An optional field to order the results by.
Returns:
(list): Returns a list of data (dict).
"""
print('Getting data...')
database = initialize_firebase()
data = get_collection(ref, database=database, order_by=order_by)
print('Found {} observations.'.format(len(data)))
if datafile:
print('Saving data...')
save_data(data, datafile)
print('Saved data to', datafile)
return data
def automatic_collection(org_id=None, env_file='.env', minutes_ago=None):
"""Automatically collect results from scientific instruments.
Args:
org_id (str): The organization ID to associate with instrument results.
env_file (str): The environment variable file, `.env` by default.
Either a `GOOGLE_APPLICATION_CREDENTIALS` or a
`CANNLYTICS_API_KEY` is needed to run the routine.
minutes_ago (int): The number of minutes in the past to restrict
recently modified files.
Returns:
(list): A list of measurements (dict) that were collected.
"""
# Try to initialize Firebase, otherwise an API key will be used.
try:
env = environ.Env()
env.read_env(env_file)
credentials = env('GOOGLE_APPLICATION_CREDENTIALS')
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials
initialize_firebase()
except:
pass
# Get the organization ID from the .env file if not specified.
if not org_id:
org_id = env('CANNLYTICS_ORGANIZATION_ID')
# Format the last modified time cut-off as a datetime.
last_modified_at = None
if minutes_ago:
last_modified_at = datetime.now() - timedelta(minutes=minutes_ago)
# Get the instruments, trying Firestore, then the API.
try:
ref = f'organizations/{org_id}/instruments'
instrument_data = get_collection(ref)
except:
api_key = env('CANNLYTICS_API_KEY')
headers = {
'Authorization': 'Bearer %s' % api_key,
'Content-type': 'application/json',
}
url = f'{API_BASE}/instruments?organization_id={org_id}'
response = requests.get(url, headers=headers)
instrument_data = response.json()['data']
# Iterate over instruments, collecting measurements.
measurements = []
for instrument in instrument_data:
# Iterate over analyses that the instrument may be running.
try:
analyses = instrument.get('analyses', '').split(',')
except AttributeError:
continue # FIXME: Handle missing analyses more elegantly.
analyses = [x.strip() for x in analyses]
for n in range(len(analyses)):
# Optional: Handle multiple data paths more elegantly.
analysis = analyses[n]
try:
data_paths = instrument['data_path'].split(',')
except AttributeError:
continue # No data path.
data_paths = [x.strip() for x in data_paths]
data_path = data_paths[n]
if not data_path:
continue
# Identify the analysis being run and identify the import routine.
# Optional: Identify more elegantly.
if 'micro' in analysis or 'MICR' in analysis:
import_method = globals()['import_micro']
elif 'metal' in analysis or 'HEAV' in analysis:
import_method = globals()['import_heavy_metals']
else:
import_method = globals()['import_results']
# Search for recently modified files in the instrument directory
# and parse any recently modified file.
for root, _, filenames in os.walk(data_path):
for filename in filenames:
if filename.endswith('.xlsx') or filename.endswith('.xls'):
data_file = os.path.join(root, filename)
modifed_at = os.stat(data_file).st_mtime
# FIXME: Ensure date restriction works.
if last_modified_at:
if modifed_at < last_modified_at:
continue
samples = import_method(data_file)
if isinstance(samples, dict):
sample_data = {**instrument, **samples}
measurements.append(sample_data)
else:
for sample in samples:
sample_data = {**instrument, **sample}
measurements.append(sample_data)
# Upload measurement data to Firestore.
now = datetime.now()
updated_at = now.isoformat()
for measurement in measurements:
try:
measurement['sample_id'] = measurement['sample_name']
except:
continue # Already has `sample_id`.
# TODO: Format a better measurement ID.
measurement_id = measurement.get(
'acq_inj_time') # E.g. 12-Jun-21, 15:21:07
if not measurement_id:
# measurement_id = now.strftime('%d-%b-%y-%H-%M-%S') + '_' + str(measurement['sample_id'])
measurement_id = measurement['sample_id']
else:
try:
measurement_id = measurement_id.replace(',', '').replace(
' ', '-').replace(':', '-')
except AttributeError:
pass
measurement_id = str(measurement_id) + '_' + str(
measurement['sample_id'])
measurement['measurement_id'] = measurement_id
measurement['updated_at'] = updated_at
ref = f'organizations/{org_id}/measurements/{measurement_id}'
try:
update_document(ref, measurement)
except:
url = f'{API_BASE}/measurements/{measurement_id}?organization_id={org_id}'
response = requests.post(url, json=measurement, headers=headers)
print('Uploaded measurement:', ref)
# Upload result data to Firestore
for result in measurement['results']:
analyte = result['analyte']
result_id = f'{measurement_id}_{analyte}'
result['sample_id'] = measurement['sample_name']
result['result_id'] = result_id
result['measurement_id'] = measurement_id
result['updated_at'] = updated_at
ref = f'organizations/{org_id}/results/{result_id}'
try:
update_document(ref, result)
except:
url = f'{API_BASE}/results/{result_id}?organization_id={org_id}'
response = requests.post(url, json=result, headers=headers)
print('Uploaded result:', ref)
# Return the measurements
return measurements
#------------------------------------------------------------------
# Initialization
#------------------------------------------------------------------
# Initialize the current time.
now = datetime.now()
current_time = now.isoformat()
current_date = now.strftime('%m/%d/%Y')
today = current_time[:10]
# Initialize Firebase.
config = dotenv_values('../../../.env')
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = config[
'GOOGLE_APPLICATION_CREDENTIALS']
db = fb.initialize_firebase()
# Initialize a Metrc client.
vendor_api_key = config['METRC_TEST_VENDOR_API_KEY']
user_api_key = config['METRC_TEST_USER_API_KEY']
track = metrc.authorize(vendor_api_key, user_api_key)
print('--------------------------------------------')
print('Performing Metrc Validation Test')
print(current_time)
print('--------------------------------------------')
#------------------------------------------------------------------
# Facilities
#------------------------------------------------------------------
# - Number of new clients.
# Upload the summary statistics.
update_document(f'organizations/{org_id}/, summary_stats)
# TODO: Create figures.
# TODO: Create LaTeX tables and text.
# Optional: Email the report.
return summary_stats
if __name__ == '__main__':
# Initialize Firebase.
env = environ.Env()
env.read_env('../../../.env')
credentials = env('GOOGLE_APPLICATION_CREDENTIALS')
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials
db = firebase.initialize_firebase()
# Create lab report for a given company.
org_id = 'test-company'
summary_stats = create_weekly_lab_report(org_id)
delete (bool): Wether or not to delete the original documents,
`False` by default.
"""
print(f'Moving documents from {ref} to {dest}...')
docs = get_collection(ref)
for doc in docs:
update_document(dest + '/' + doc['id'], doc)
if delete:
delete_document(ref + '/' + doc['id'])
print(f'Moved all documents in {ref} to {dest}.')
if __name__ == '__main__':
# Initialize Firebase.
try:
config = dotenv_values('../../.env')
credentials = config['GOOGLE_APPLICATION_CREDENTIALS']
except KeyError:
config = dotenv_values('.env')
credentials = config['GOOGLE_APPLICATION_CREDENTIALS']
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials
initialize_firebase()
# Read arguments from the command line.
original_ref = sys.argv[1]
destination_ref = sys.argv[2]
# Move the collection.
move_collection(original_ref, destination_ref)
data = {'success': True, 'message': 'Organization statistics updated.'}
return (data, 200, headers)
if __name__ == '__main__':
print('Testing...')
import os
import environ
# Initialize Firebase.
env = environ.Env()
env.read_env('../../.env')
credentials = env('GOOGLE_APPLICATION_CREDENTIALS')
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials
db = initialize_firebase()
# Get parameters.
org_id = 'test-company'
# For each data model, calculate stats
ref = f'organizations/{org_id}/data_models'
data_models = get_collection(ref)
for data_model in data_models[:1]:
# Calculate stats for given data model.
key = data_model['key']
data_ref = f'organizations/{org_id}/{key}'
data = get_collection(data_ref)
# TODO: Daily stats
def test_firestore():
"""Test Firestore functions by managing a test document."""
# Initialize Firebase.
env = environ.Env()
env.read_env('../.env')
credentials = env('GOOGLE_APPLICATION_CREDENTIALS')
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials
db = firebase.initialize_firebase()
# Create a collection reference.
col_ref = firebase.create_reference(db, 'tests')
assert isinstance(col_ref, CollectionReference) == True
# Create a document reference.
doc_ref = firebase.create_reference(db, 'tests/firebase_test')
assert isinstance(doc_ref, DocumentReference) == True
# Create a document.
firebase.update_document('tests/firebase_test', {'user': '******'})
# Update a document.
firebase.update_document('tests/firebase_test', {'test': 'firebase_test'})
# Get the document.
data = firebase.get_document('tests/firebase_test')
assert data['user'] == 'CannBot'
assert data['test'] == 'firebase_test'
# Get a collection.
filters = [{'key': 'test', 'operation': '==', 'value': 'firebase_test'}]
docs = firebase.get_collection('tests', filters=filters)
assert docs[0]['test'] == 'firebase_test'
# Add an element to an array in a document.
firebase.add_to_array('tests/firebase_test', 'likes', 'Testing')
data = firebase.get_document('tests/firebase_test')
assert 'Testing' in data['likes']
# Remove an element from an array in a document.
firebase.remove_from_array('tests/firebase_test', 'likes', 'Testing')
data = firebase.get_document('tests/firebase_test')
assert 'Testing' not in data['likes']
# Increment a value in a document.
firebase.increment_value('tests/firebase_test', 'runs')
data = firebase.get_document('tests/firebase_test')
assert data['runs'] > 0
# Import .csv data to Firestore.
ref = 'tests/test_collections/licensees'
data_file = './assets/data/licensees_partial.csv'
firebase.import_data(db, ref, data_file)
# TODO: Test import .xlsx data to Firestore.
# TODO: Test import .txt data to Firestore.
# Export data to .csv from Firestore.
output_csv_file = './assets/data/licensees_test.csv'
output_xlsx_file = './assets/data/licensees_test.xlsx'
firebase.export_data(db, ref, output_csv_file)
# Export data to .xlsx from Firestore.
firebase.export_data(db, ref, output_xlsx_file)
