def test_publish_data(dataset): """Publish a dataset on the data market.""" # Initialize Ocean market. ocean, config = initialize_ocean_market() # Mint a test OCEAN. os.environ['FACTORY_DEPLOYER_PRIVATE_KEY'] = config['FACTORY_DEPLOYER_PRIVATE_KEY'] mint_fake_OCEAN(ocean.config) # Publish a dataset. data_token, asset = market.publish_data( ocean, config.get(SELLER_KEY), files=dataset['files'], name=dataset['datatoken_name'], symbol=dataset['datatoken_symbol'], author=dataset['author'], data_license=dataset['license'], ) # Upload the datatoken and asset information. os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = config['GOOGLE_APPLICATION_CREDENTIALS'] initialize_firebase() ref = f'public/market/datasets/{asset.asset_id}' entry = {**dataset, **asset.as_dictionary()} update_document(ref, entry) return data_token, asset
def upload_contributors(org_name): """Get Github contributors and save them to Firestore. Args: org_name (str): The name of a GitHub organization. Returns: (list): A list of users (dict). """ users = [] client = Github() org = client.get_organization(org_name) repos = org.get_repos() initialize_firebase() for repo in repos: contributors = repo.get_contributors() for user in contributors: if user.name not in users: users.append(user.name) data = { 'company': user.company, 'description': user.bio, 'name': user.name, 'location': user.location, 'image': user.avatar_url, 'url': user.html_url, 'slug': user.login, } update_document(f'public/contributors/contributor_data/{user.id}', data) return users
def get_context_data(self, **kwargs): """Get context that is used on all pages. The context is retrieved dynamically from the app's state. The user's permissions are verified on every request. User-specific context and data is returned depending on the page. Information about data models is provided to all pages.""" context = super(BaseMixin, self).get_context_data(**kwargs) context = get_page_context(self.kwargs, context) initialize_firebase() context = get_user_data(self.request, context) context = get_page_data(context) save_analytics(self.request, context) return context
def test_storage(): """Test Firebase Storage by managing a test file.""" # Get the path to your service account. from dotenv import dotenv_values config = dotenv_values('./env') key_path = config['GOOGLE_APPLICATION_CREDENTIALS'] bucket_name = config['FIREBASE_STORAGE_BUCKET'] # Initialize Firebase. firebase.initialize_firebase(key_path, bucket_name) # Define file names. bucket_folder = 'tests/assets/pdfs' destination_blob_name = 'tests/assets/pdfs/pandas_cheat_sheet.pdf' local_folder = './assets/pdfs' source_file_name = './assets/pdfs/Pandas_Cheat_Sheet.pdf' download_folder = './assets/downloads/pdfs' download_file_name = './assets/downloads/pdfs/Pandas_Cheat_Sheet.pdf' file_name = 'pandas_cheat_sheet.pdf' file_copy = 'pandas_cheat_sheet_copy.pdf' newfile_name = 'tests/assets/pdfs/' + file_copy # Upload a file to a Firebase Storage bucket, with and without bucket name. firebase.upload_file(destination_blob_name, source_file_name) firebase.upload_file(destination_blob_name, source_file_name, bucket_name) # Upload all files in a folder to a Firebase Storage bucket, with and without bucket name. firebase.upload_files(bucket_folder, local_folder) firebase.upload_files(bucket_folder, local_folder, bucket_name) # List all files in the Firebase Storage bucket folder, with and without bucket name. files = firebase.list_files(bucket_folder) assert isinstance(files, list) files = firebase.list_files(bucket_folder, bucket_name) assert isinstance(files, list) # Download a file from Firebase Storage, with and without bucket name. firebase.download_file(destination_blob_name, download_file_name) firebase.download_file(destination_blob_name, download_file_name, bucket_name) # Download all files in a given Firebase Storage folder, with and without bucket name. firebase.download_files(bucket_folder, download_folder) firebase.download_files(bucket_folder, download_folder, bucket_name) # Rename a file in the Firebase Storage bucket, with and without bucket name. firebase.rename_file(bucket_folder, file_name, newfile_name) firebase.rename_file(bucket_folder, file_name, newfile_name, bucket_name) # Delete a file from the Firebase Storage bucket, with and without bucket name. firebase.delete_file(bucket_folder, file_copy) firebase.delete_file(bucket_folder, file_copy, bucket_name)
def login(request, *args, **argv): #pylint: disable=unused-argument """Functional view to create a user session. Optional: Ensure that the request succeeds on the client! """ try: print('Logging user in...') authorization = request.headers.get('Authorization', '') token = authorization.split(' ').pop() if not token: # return HttpResponse(status=401) message = 'Authorization token not provided in the request header.' return JsonResponse({ 'error': True, 'message': message }, status=401) initialize_firebase() print('Initialized Firebase.') # Set session cookie in a cookie in the response. # response = HttpResponse(status=200) response = JsonResponse({'success': True}, status=200) # Optional: Let user specify cookie duration? # expires_in = timedelta(days=5) # expires = datetime.now() + expires_in session_cookie = create_session_cookie(token) response['Set-Cookie'] = f'__session={session_cookie}; Path=/' response[ 'Cache-Control'] = 'public, max-age=300, s-maxage=900' # TODO: Set the expiration time # Save session cookie in the session. # Preferred over cookies (but cookies are still needed for production). request.session['__session'] = session_cookie # Verify the user, create a log, update the user as signed-in, # and return a response with the session cookie. claims = verify_token(token) uid = claims['uid'] print('Verified user with Firebase Authentication:', claims['email']) create_log(ref=f'users/{uid}/logs', claims=claims, action='Signed in.', log_type='auth', key='login') update_document(f'users/{uid}', {'signed_in': True}) print('Logged user sign-in in Firestore:', uid) return response except: # return HttpResponse(status=401) message = 'Authorization failed in entirety. Please contact support.' return JsonResponse({'error': True, 'message': message}, status=401)
def upload_data( file_name: str, collection: str, id_key: Optional[str] = 'id', stats_doc: Optional[str] = '', ): """ Upload a dataset to Firestore. Args: datafile (str): The path to a .json file containing the data. collection (str): The path of the collection where data will be stored. id_key (str): The key of the ID. stats_doc (str): An optional document to store statistics about the data. Returns: data (list): A list of partner data (dict). """ database = initialize_firebase() with open(file_name) as datafile: data = json.load(datafile) print('Uploading dataset...') for item in data: item['updated_at'] = datetime.now().isoformat() doc_id = item[id_key] ref = f'{collection}/{doc_id}' update_document(ref, item, database=database) print('Updated:', ref) if stats_doc: update_document(stats_doc, {'total': len(data)}, database=database) print('Updated:', stats_doc) print('Finished uploading data.') return data
def login(request): """Functional view to create a user session.""" try: # Ensure that the user passed an authorization bearer token. authorization = request.headers.get('Authorization') token = authorization.split(' ').pop() if not token: message = 'Authorization token not provided in the request header.' return JsonResponse({ 'success': False, 'message': message }, status=401) # Initialize Firebase and verify the Firebase ID token. initialize_firebase() claims = verify_token(token) uid = claims['uid'] # Create and set a session cookie in the response. cache = f'public, max-age={SESSION_COOKIE_AGE}, s-maxage={SESSION_COOKIE_AGE}' session_cookie = create_session_cookie(token) response = JsonResponse({'success': True}, status=200) response['Cache-Control'] = cache response['Set-Cookie'] = f'__session={session_cookie}; Path=/' # Also save the session cookie in the session. # Note: The session is preferred over cookies, # but cookies are currently needed for production. request.session['__session'] = session_cookie # Log the login and update the user as signed-in. update_document(f'users/{uid}', {'signed_in': True}) create_log(ref=f'users/{uid}/logs', claims=claims, action='Signed in.', log_type='auth', key='login') return response except: message = f'Authorization failed in entirety. Please contact {DEFAULT_FROM_EMAIL}' return JsonResponse({'success': False, 'message': message}, status=401)
def test_storage(): """Test Firebase Storage by managing a test file.""" # Initialize Firebase. env = environ.Env() env.read_env('../.env') credentials = env('GOOGLE_APPLICATION_CREDENTIALS') os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials firebase.initialize_firebase() # Define file names. bucket_name = 'cannlytics.appspot.com' bucket_folder = 'tests/assets/pdfs' destination_blob_name = 'tests/assets/pdfs/pandas_cheat_sheet.pdf' local_folder = './assets/pdfs' source_file_name = './assets/pdfs/Pandas_Cheat_Sheet.pdf' download_folder = './assets/downloads/pdfs' download_file_name = './assets/downloads/pdfs/Pandas_Cheat_Sheet.pdf' file_name = 'pandas_cheat_sheet.pdf' file_copy = 'pandas_cheat_sheet_copy.pdf' newfile_name = 'tests/assets/pdfs/' + file_copy # Upload a file to a Firebase Storage bucket. firebase.upload_file(bucket_name, destination_blob_name, source_file_name) # Upload all files in a folder to a Firebase Storage bucket. firebase.upload_files(bucket_name, bucket_folder, local_folder) # List all files in the Firebase Storage bucket folder. files = firebase.list_files(bucket_name, bucket_folder) assert isinstance(files, list) == True # Download a file from Firebase Storage. firebase.download_file(bucket_name, destination_blob_name, download_file_name) # Download all files in a given Firebase Storage folder. firebase.download_files(bucket_name, bucket_folder, download_folder) # Rename a file in the Firebase Storage bucket. firebase.rename_file(bucket_name, bucket_folder, file_name, newfile_name) # Delete a file from the Firebase Storage bucket. firebase.delete_file(bucket_name, bucket_folder, file_copy)
def test_auth(): """Test Firebase Authentication by managing a user.""" # Initialize Firebase env = environ.Env() env.read_env('../.env') credentials = env('GOOGLE_APPLICATION_CREDENTIALS') os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials firebase.initialize_firebase() # Create account. name = 'CannBot' email = '*****@*****.**' claims = {'organizations': ['Cannlytics']} user, _ = firebase.create_account(name, email, notification=True) # Create and get custom claims. firebase.create_custom_claims(user.uid, email=email, claims=claims) custom_claims = firebase.get_custom_claims(email) # Create custom token. token = firebase.create_custom_token(user.uid, email=None, claims=custom_claims) assert isinstance(token, bytes) == True # Get user. user = firebase.get_user(email) # Get all users. all_users = firebase.get_users() assert isinstance(all_users, list) == True # Update user. photo_url = f'https://robohash.org/{email}?set=set5' user = firebase.update_user(user, {'photo_url': photo_url}) assert user.photo_url == 'https://robohash.org/[email protected]?set=set5' # Delete a user. firebase.delete_user(user.uid)
def test_secret_manager(): """Test Secret Manager by creating a secret, updating the version of the secret, then deleting the secret.""" # Initialize Firebase env = environ.Env() env.read_env('../../.env') credentials = env('GOOGLE_APPLICATION_CREDENTIALS') os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials firebase.initialize_firebase() # Mock license. license_data = { 'license_number': 'test', 'license_type': 'lab', 'state': 'OK', 'user_api_key': '123', } license_number = license_data['license_number'] user_api_key = license_data['user_api_key'] # Create a secret. project_id = 'cannlytics' secret_id = f'{license_number}-secret' try: version_id = firebase.create_secret(project_id, secret_id, user_api_key) except: # AlreadyExists error pass # Secret may already be created. # Add the secret's secret data. secret = firebase.add_secret_version(project_id, secret_id, user_api_key) version_id = secret.split('/')[-1] # Get the secret. # In production the secret ID and version ID are stored with the license data. user_api_key = firebase.access_secret_version(project_id, secret_id, version_id) assert user_api_key == license_data['user_api_key']
def test_create_log(): """Test create a log.""" # Initialize Firebase. env = environ.Env() env.read_env('../.env') credentials = env('GOOGLE_APPLICATION_CREDENTIALS') os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials db = firebase.initialize_firebase() # Create a test log. user = { 'uid': 'test', 'display_name': 'CannBot', 'email': '*****@*****.**', 'photo_url': 'https://robohash.org/[email protected]', } logs = 'tests/test_collections/logs' firebase.create_log(logs, user, 'Test log.', 'test', 'create_log')
def get_data( ref: str, datafile: Optional[str] = '', order_by: Optional[str] = '', ) -> List[dict]: """Get a dataset saved in Firestore. Args: ref (str): The reference to a collection. datafile (str): Save the data locally to the project's `.datasets` if a data file is given. order_by (str): An optional field to order the results by. Returns: (list): Returns a list of data (dict). """ print('Getting data...') database = initialize_firebase() data = get_collection(ref, database=database, order_by=order_by) print('Found {} observations.'.format(len(data))) if datafile: print('Saving data...') save_data(data, datafile) print('Saved data to', datafile) return data
def automatic_collection(org_id=None, env_file='.env', minutes_ago=None): """Automatically collect results from scientific instruments. Args: org_id (str): The organization ID to associate with instrument results. env_file (str): The environment variable file, `.env` by default. Either a `GOOGLE_APPLICATION_CREDENTIALS` or a `CANNLYTICS_API_KEY` is needed to run the routine. minutes_ago (int): The number of minutes in the past to restrict recently modified files. Returns: (list): A list of measurements (dict) that were collected. """ # Try to initialize Firebase, otherwise an API key will be used. try: env = environ.Env() env.read_env(env_file) credentials = env('GOOGLE_APPLICATION_CREDENTIALS') os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials initialize_firebase() except: pass # Get the organization ID from the .env file if not specified. if not org_id: org_id = env('CANNLYTICS_ORGANIZATION_ID') # Format the last modified time cut-off as a datetime. last_modified_at = None if minutes_ago: last_modified_at = datetime.now() - timedelta(minutes=minutes_ago) # Get the instruments, trying Firestore, then the API. try: ref = f'organizations/{org_id}/instruments' instrument_data = get_collection(ref) except: api_key = env('CANNLYTICS_API_KEY') headers = { 'Authorization': 'Bearer %s' % api_key, 'Content-type': 'application/json', } url = f'{API_BASE}/instruments?organization_id={org_id}' response = requests.get(url, headers=headers) instrument_data = response.json()['data'] # Iterate over instruments, collecting measurements. measurements = [] for instrument in instrument_data: # Iterate over analyses that the instrument may be running. try: analyses = instrument.get('analyses', '').split(',') except AttributeError: continue # FIXME: Handle missing analyses more elegantly. analyses = [x.strip() for x in analyses] for n in range(len(analyses)): # Optional: Handle multiple data paths more elegantly. analysis = analyses[n] try: data_paths = instrument['data_path'].split(',') except AttributeError: continue # No data path. data_paths = [x.strip() for x in data_paths] data_path = data_paths[n] if not data_path: continue # Identify the analysis being run and identify the import routine. # Optional: Identify more elegantly. if 'micro' in analysis or 'MICR' in analysis: import_method = globals()['import_micro'] elif 'metal' in analysis or 'HEAV' in analysis: import_method = globals()['import_heavy_metals'] else: import_method = globals()['import_results'] # Search for recently modified files in the instrument directory # and parse any recently modified file. for root, _, filenames in os.walk(data_path): for filename in filenames: if filename.endswith('.xlsx') or filename.endswith('.xls'): data_file = os.path.join(root, filename) modifed_at = os.stat(data_file).st_mtime # FIXME: Ensure date restriction works. if last_modified_at: if modifed_at < last_modified_at: continue samples = import_method(data_file) if isinstance(samples, dict): sample_data = {**instrument, **samples} measurements.append(sample_data) else: for sample in samples: sample_data = {**instrument, **sample} measurements.append(sample_data) # Upload measurement data to Firestore. now = datetime.now() updated_at = now.isoformat() for measurement in measurements: try: measurement['sample_id'] = measurement['sample_name'] except: continue # Already has `sample_id`. # TODO: Format a better measurement ID. measurement_id = measurement.get( 'acq_inj_time') # E.g. 12-Jun-21, 15:21:07 if not measurement_id: # measurement_id = now.strftime('%d-%b-%y-%H-%M-%S') + '_' + str(measurement['sample_id']) measurement_id = measurement['sample_id'] else: try: measurement_id = measurement_id.replace(',', '').replace( ' ', '-').replace(':', '-') except AttributeError: pass measurement_id = str(measurement_id) + '_' + str( measurement['sample_id']) measurement['measurement_id'] = measurement_id measurement['updated_at'] = updated_at ref = f'organizations/{org_id}/measurements/{measurement_id}' try: update_document(ref, measurement) except: url = f'{API_BASE}/measurements/{measurement_id}?organization_id={org_id}' response = requests.post(url, json=measurement, headers=headers) print('Uploaded measurement:', ref) # Upload result data to Firestore for result in measurement['results']: analyte = result['analyte'] result_id = f'{measurement_id}_{analyte}' result['sample_id'] = measurement['sample_name'] result['result_id'] = result_id result['measurement_id'] = measurement_id result['updated_at'] = updated_at ref = f'organizations/{org_id}/results/{result_id}' try: update_document(ref, result) except: url = f'{API_BASE}/results/{result_id}?organization_id={org_id}' response = requests.post(url, json=result, headers=headers) print('Uploaded result:', ref) # Return the measurements return measurements
#------------------------------------------------------------------ # Initialization #------------------------------------------------------------------ # Initialize the current time. now = datetime.now() current_time = now.isoformat() current_date = now.strftime('%m/%d/%Y') today = current_time[:10] # Initialize Firebase. config = dotenv_values('../../../.env') os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = config[ 'GOOGLE_APPLICATION_CREDENTIALS'] db = fb.initialize_firebase() # Initialize a Metrc client. vendor_api_key = config['METRC_TEST_VENDOR_API_KEY'] user_api_key = config['METRC_TEST_USER_API_KEY'] track = metrc.authorize(vendor_api_key, user_api_key) print('--------------------------------------------') print('Performing Metrc Validation Test') print(current_time) print('--------------------------------------------') #------------------------------------------------------------------ # Facilities #------------------------------------------------------------------
# - Number of new clients. # Upload the summary statistics. update_document(f'organizations/{org_id}/, summary_stats) # TODO: Create figures. # TODO: Create LaTeX tables and text. # Optional: Email the report. return summary_stats if __name__ == '__main__': # Initialize Firebase. env = environ.Env() env.read_env('../../../.env') credentials = env('GOOGLE_APPLICATION_CREDENTIALS') os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials db = firebase.initialize_firebase() # Create lab report for a given company. org_id = 'test-company' summary_stats = create_weekly_lab_report(org_id)
delete (bool): Wether or not to delete the original documents, `False` by default. """ print(f'Moving documents from {ref} to {dest}...') docs = get_collection(ref) for doc in docs: update_document(dest + '/' + doc['id'], doc) if delete: delete_document(ref + '/' + doc['id']) print(f'Moved all documents in {ref} to {dest}.') if __name__ == '__main__': # Initialize Firebase. try: config = dotenv_values('../../.env') credentials = config['GOOGLE_APPLICATION_CREDENTIALS'] except KeyError: config = dotenv_values('.env') credentials = config['GOOGLE_APPLICATION_CREDENTIALS'] os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials initialize_firebase() # Read arguments from the command line. original_ref = sys.argv[1] destination_ref = sys.argv[2] # Move the collection. move_collection(original_ref, destination_ref)
data = {'success': True, 'message': 'Organization statistics updated.'} return (data, 200, headers) if __name__ == '__main__': print('Testing...') import os import environ # Initialize Firebase. env = environ.Env() env.read_env('../../.env') credentials = env('GOOGLE_APPLICATION_CREDENTIALS') os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials db = initialize_firebase() # Get parameters. org_id = 'test-company' # For each data model, calculate stats ref = f'organizations/{org_id}/data_models' data_models = get_collection(ref) for data_model in data_models[:1]: # Calculate stats for given data model. key = data_model['key'] data_ref = f'organizations/{org_id}/{key}' data = get_collection(data_ref) # TODO: Daily stats
def test_firestore(): """Test Firestore functions by managing a test document.""" # Initialize Firebase. env = environ.Env() env.read_env('../.env') credentials = env('GOOGLE_APPLICATION_CREDENTIALS') os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials db = firebase.initialize_firebase() # Create a collection reference. col_ref = firebase.create_reference(db, 'tests') assert isinstance(col_ref, CollectionReference) == True # Create a document reference. doc_ref = firebase.create_reference(db, 'tests/firebase_test') assert isinstance(doc_ref, DocumentReference) == True # Create a document. firebase.update_document('tests/firebase_test', {'user': '******'}) # Update a document. firebase.update_document('tests/firebase_test', {'test': 'firebase_test'}) # Get the document. data = firebase.get_document('tests/firebase_test') assert data['user'] == 'CannBot' assert data['test'] == 'firebase_test' # Get a collection. filters = [{'key': 'test', 'operation': '==', 'value': 'firebase_test'}] docs = firebase.get_collection('tests', filters=filters) assert docs[0]['test'] == 'firebase_test' # Add an element to an array in a document. firebase.add_to_array('tests/firebase_test', 'likes', 'Testing') data = firebase.get_document('tests/firebase_test') assert 'Testing' in data['likes'] # Remove an element from an array in a document. firebase.remove_from_array('tests/firebase_test', 'likes', 'Testing') data = firebase.get_document('tests/firebase_test') assert 'Testing' not in data['likes'] # Increment a value in a document. firebase.increment_value('tests/firebase_test', 'runs') data = firebase.get_document('tests/firebase_test') assert data['runs'] > 0 # Import .csv data to Firestore. ref = 'tests/test_collections/licensees' data_file = './assets/data/licensees_partial.csv' firebase.import_data(db, ref, data_file) # TODO: Test import .xlsx data to Firestore. # TODO: Test import .txt data to Firestore. # Export data to .csv from Firestore. output_csv_file = './assets/data/licensees_test.csv' output_xlsx_file = './assets/data/licensees_test.xlsx' firebase.export_data(db, ref, output_csv_file) # Export data to .xlsx from Firestore. firebase.export_data(db, ref, output_xlsx_file)