def compile_map(folder_path, file_suffix='', json_model=False, pythonic=False):
from os import path
from labpack.records.settings import load_settings
file_map = {}
file_list = compile_list(folder_path, file_suffix)
for file_path in file_list:
file_details = load_settings(file_path)
file_key = path.split(file_path)[1].replace(file_suffix, '')
if pythonic:
file_key = file_key.replace(' ', '_').replace('-', '_').lower()
if json_model:
from jsonmodel.validators import jsonModel
file_map[file_key] = jsonModel(file_details)
# add any schema in metadata
if 'schema' in file_map[file_key].metadata.keys():
metadata_details = file_map[file_key].metadata
metadata_key = file_key + '-metadata'
if pythonic:
metdata_key = metadata_key.replace(' ', '_').replace(
'-', '_').lower()
file_map[metadata_key] = jsonModel(metadata_details)
else:
file_map[file_key] = file_details
return file_map
def __init__(self, message_schema, case_schema, request_schema, response_schema, question_sequence):
self.messageModel = jsonModel(message_schema)
self.caseModel = jsonModel(case_schema)
self.requestModel = jsonModel(request_schema)
self.responseModel = jsonModel(response_schema)
self.questions = question_sequence
def compile_model(command_schema, cli_schema):
'''
a method to create a jsonmodel object for command fields with cli metadata
:param command_schema: dictionary with jsonmodel valid schema for command arguments
:param cli_schema: dictionary with jsonmodel valid schema for command line interface metadata
:return: jsonmodel object for command fields
'''
# validate model structure
if not 'components' in command_schema.keys():
command_schema['components'] = {}
if not 'metadata' in command_schema.keys():
command_schema['metadata'] = {}
# construct cli_model
cli_model = jsonModel(cli_schema)
# inject cli fields into the metadata for each field
for key, value in command_schema['schema'].items():
field_key = '.%s' % key
if not field_key in command_schema['components'].keys():
command_schema['components'][field_key] = {}
if not 'field_metadata' in command_schema['components'][
field_key].keys():
command_schema['components'][field_key]['field_metadata'] = {}
cli_fields = command_schema['components'][field_key]['field_metadata']
command_schema['components'][field_key][
'field_metadata'] = cli_model.ingest(**cli_fields)
# inject cli fields into metadata field
command_schema['metadata'] = cli_model.ingest(**command_schema['metadata'])
return jsonModel(command_schema)
def __init__(self, message_schema, case_schema, request_schema,
response_schema, question_sequence):
self.messageModel = jsonModel(message_schema)
self.caseModel = jsonModel(case_schema)
self.requestModel = jsonModel(request_schema)
self.responseModel = jsonModel(response_schema)
self.questions = question_sequence
def __init__(self, client_id, client_secret, auth_endpoint, token_endpoint, redirect_uri, request_mimetype='', status_endpoint='', requests_handler=None, error_map=None):
''' the initialization method for oauth2 client class
:param client_id: string with client id registered for app with service
:param client_secret: string with client secret registered for app with service
:param auth_endpoint: string with service endpoint for authorization code requests
:param token_endpoint: string with service endpoint for token post requests
:param redirect_uri: string with url for redirect callback registered with service
:param request_mimetype: [optional] string with mimetype for token post requests
:param status_endpoint: [optional] string with service endpoint to retrieve status of token
:param requests_handler: [optional] callable that handles requests errors
:param error_map: [optional] dictionary with key value strings for service error msgs
'''
title = '%s.__init__' % self.__class__.__name__
# construct class field model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# construct class object models
object_models = {}
for key, value in self._class_objects.items():
object_models[key] = jsonModel(value)
from labpack.compilers.objects import _method_constructor
self.objects = _method_constructor(object_models)
# validate inputs
input_fields = {
'client_id': client_id,
'client_secret': client_secret,
'auth_endpoint': auth_endpoint,
'token_endpoint': token_endpoint,
'redirect_uri': redirect_uri,
'request_mimetype': request_mimetype,
'status_endpoint': status_endpoint,
'error_map': error_map
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct class properties
self.client_id = client_id
self.client_secret = client_secret
self.auth_endpoint = auth_endpoint
self.token_endpoint = token_endpoint
self.redirect_uri = redirect_uri
self.request_mimetype = request_mimetype
self.status_endpoint = status_endpoint
# construct handlers
self.requests_handler = requests_handler
self.error_map = error_map
def retrieve_settings(model_path, file_path, secret_key=''):
# validate input
title = 'retrieve_settings'
from jsonmodel.validators import jsonModel
model_details = load_settings(model_path)
settings_model = jsonModel(model_details)
# try to load settings from file
file_settings = {}
try:
file_settings = load_settings(file_path, secret_key)
except:
pass
# retrieve environmental variables
environ_var = ingest_environ()
# construct settings details from file and environment
settings_details = settings_model.ingest(**{})
for key in settings_model.schema.keys():
if key.upper() in environ_var.keys():
settings_details[key] = environ_var[key.upper()]
elif key in file_settings.keys():
settings_details[key] = file_settings[key]
return settings_details
def __init__(self, access_token, service_scope, usage_client=None, requests_handler=None):
''' initialization method for moves client class
:param access_token: string with access token for user provided by moves oauth
:param service_scope: dictionary with service type permissions
:param usage_client: callable that records usage data
:param requests_handler: callable that handles requests errors
'''
title = '%s.__init__' % self.__class__.__name__
# construct class field model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# construct client attributes
object_title = '%s(access_token=%s)' % (title, str(access_token))
self.access_token = self.fields.validate(access_token, '.access_token', object_title)
object_title = '%s(service_scope=[...])' % title
self.service_scope = self.fields.validate(service_scope, '.service_scope', object_title)
self.endpoint = self.fields.schema['api_endpoint']
# construct handlers
self.moves_handler = movesHandler(usage_client)
self.requests_handler = requests_handler
def __init__(self, api_key, allow_fees=False, usage_client=None, requests_handler=None):
'''
a method to initialize the mandrill client class
:param api_key: string with api key generated by mandrill
:param allow_fees: [optional] boolean to allow additional fees
:param usage_client: callable that records usage data
:param requests_handler: callable that handles requests errors
'''
title = '%s.__init__' % self.__class__.__name__
# construct class field model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'api_key': api_key
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct class properties
self.api_endpoint = self.fields.schema['api_endpoint']
self.api_key = api_key
self.allow_fees = allow_fees
# construct method handlers
self.service_handler = mandrillHandler(usage_client)
self.requests_handler = requests_handler
def __init__(self, keyspace_name, table_name, record_schema, cassandra_session, replication_strategy=None):
title = '%s.__init__' % self.__class__.__name__
# construct fields model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'keyspace_name': keyspace_name,
'table_name': table_name,
'record_schema': record_schema,
'replication_strategy': replication_strategy
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# validate cassandra session
from sys import path as sys_path
sys_path.append(sys_path.pop(0))
from cassandra.cluster import Session
sys_path.insert(0, sys_path.pop())
if not isinstance(cassandra_session, Session):
raise ValueError('%s(cassandra_session) must be a cassandra.cluster.Session datatype.' % title)
self.session = cassandra_session
def __init__(self, virtualbox_name='', verbose=False):
'''
a method to initialize the dockerClient class
:param virtualbox_name: [optional] string with name of virtualbox image
:return: dockerClient object
'''
title = '%s.__init__' % self.__class__.__name__
# construct super
super(dockerClient, self).__init__()
# construct fields model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'virtualbox_name': virtualbox_name
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct properties
self.vbox = virtualbox_name
self.verbose = verbose
# construct localhost
from labpack.platforms.localhost import localhostClient
self.localhost = localhostClient()
# verbosity
if self.verbose:
print('Checking docker installation...', end='', flush=True)
# validate docker installation
self._validate_install()
if self.verbose:
print('.', end='', flush=True)
# validate virtualbox installation
self.vbox_running = self._validate_virtualbox()
if self.verbose:
print('.', end='', flush=True)
# set virtualbox variables
if self.vbox_running:
self._set_virtualbox()
if self.verbose:
print('.', end='', flush=True)
if self.verbose:
print(' done.')
def ingest_environ(model_path=''):
''' a method to convert environment variables to a python dictionary
:param model_path: [optional] string with path to jsonmodel of data to ingest
:return: dictionary with environmental variables
NOTE: if a model is provided, then only those fields in the model will be
added to the output and the value of any environment variable which
matches the uppercase name of each field in the model will be added
to the dictionary if its value is valid according to the model. if
a value is not valid, the method will throw a InputValidationError
'''
# convert environment variables into json typed data
from os import environ, path
typed_dict = {}
environ_variables = dict(environ)
for key, value in environ_variables.items():
if value.lower() == 'true':
typed_dict[key] = True
elif value.lower() == 'false':
typed_dict[key] = False
elif value.lower() == 'null':
typed_dict[key] = None
elif value.lower() == 'none':
typed_dict[key] = None
else:
try:
try:
typed_dict[key] = int(value)
except:
typed_dict[key] = float(value)
except:
typed_dict[key] = value
# feed environment variables through model
if model_path:
if not path.exists(model_path):
raise ValueError('%s is not a valid file path.' % model_path)
model_dict = load_settings(model_path)
from jsonmodel.validators import jsonModel
model_object = jsonModel(model_dict)
default_dict = model_object.ingest(**{})
for key in default_dict.keys():
if key.upper() in typed_dict:
valid_kwargs = {
'input_data': typed_dict[key.upper()],
'object_title': 'Environment variable %s' % key.upper(),
'path_to_root': '.%s' % key
}
default_dict[key] = model_object.validate(**valid_kwargs)
return default_dict
return typed_dict
def __init__(self, access_token, collection_name=''):
'''
a method to initialize the driveClient class
:param access_token: string with oauth2 access token for users account
:param collection_name: [optional] string with name of collection for import
'''
title = '%s.__init__' % self.__class__.__name__
# construct input validation model
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'access_token': access_token,
'collection_name': collection_name
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct access token
self.access_token = access_token
# construct drive client
import httplib2
from googleapiclient import discovery
from oauth2client.client import AccessTokenCredentials
google_credentials = AccessTokenCredentials(self.access_token, 'my-user-agent/1.0')
google_http = httplib2.Http()
google_http = google_credentials.authorize(google_http)
google_drive = discovery.build('drive', 'v3', http=google_http)
self.drive = google_drive.files()
# construct collection properties
self.permissions_write = True
self.permissions_content = True
self.drive_space = 'drive'
self.space_id = ''
if collection_name:
self.collection_name = collection_name
else:
self.collection_name = 'My Drive'
# construct file object
from labpack import __module__
from jsonmodel.loader import jsonLoader
drive_rules = jsonLoader(__module__, 'storage/google/drive-rules.json')
self.object_file = drive_rules['file_object']
# validate access token
self._validate_token()
def inject_defaults(command_schema, default_schema):
default_model = jsonModel(default_schema)
for key, value in default_model.schema.items():
if not key in command_schema['schema'].keys():
command_schema['schema'][key] = value
for key, value in default_model.components.items():
if not key in command_schema['components'].keys():
command_schema['components'][key] = value
return command_schema
def __init__(self, collection_name='', prod_name='', org_name='', root_path=''):
''' initialization method of appdata client class
:param collection_name: [optional] string with name of collection to store records
:param prod_name: [optional] string with name of application product
:param org_name: [optional] string with name of organization behind product
:param root_path: [optional] string with path to root of collections (defaults to user home)
'''
title = '%s.__init__' % self.__class__.__name__
# add localhost property to class
self.localhost = localhostClient()
# construct input validation model
self.fields = jsonModel(self._class_fields)
# validate inputs
if not collection_name:
collection_name = 'User Data'
else:
collection_name = self.fields.validate(collection_name, '.collection_name')
if not org_name:
org_name = __team__
else:
org_name = self.localhost.fields.validate(org_name, '.org_name')
if not prod_name:
prod_name = __module__
else:
prod_name = self.localhost.fields.validate(prod_name, '.prod_name')
# construct collection name
from copy import deepcopy
self.collection_name = deepcopy(collection_name)
# construct app folder
if not root_path:
self.app_folder = self.localhost.app_data(org_name=org_name, prod_name=prod_name)
else:
root_path = self.fields.validate(root_path, '.root_path')
if os.path.exists(root_path):
if not os.path.isdir(root_path):
raise ValueError('%s(root_path="%s") is an existing file.' % (title, root_path))
self.app_folder = os.path.abspath(root_path)
# validate existence of file data folder in app data (or create)
if self.localhost.os in ('Linux', 'FreeBSD', 'Solaris'):
collection_name = collection_name.replace(' ', '-').lower()
self.collection_folder = os.path.join(self.app_folder, collection_name)
self.fields.validate(self.collection_folder, '.record_key_path')
if not os.path.exists(self.collection_folder):
os.makedirs(self.collection_folder)
def __init__(self, usage_client=None):
''' initialization method for dropbox handler class
:param usage_client: callable that records usage data
'''
# construct class field model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# construct initial methods
self.rate_limits = self.fields.schema['rate_limits']
self.usage_client = usage_client
def __init__(self, client_id, client_secret, retrieve_details=True, sandbox=False, requests_handler=None, usage_client=None):
''' the initialization method for the capital one client
:param client_id: string with client id registered for app with service
:param client_secret: string with client secret registered for app with service
:param retrieve_details: boolean to automatically retrieve, store and refresh account details
:param sandbox: boolean to send requests to test sandbox
:param requests_handler: callable that handles requests errors
:param usage_client: callable that records usage data
'''
title = '%s.__init__' % self.__class__.__name__
# construct class field model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'client_id': client_id,
'client_secret': client_secret,
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct class properties
self.client_id = client_id
self.client_secret = client_secret
self.sandbox = sandbox
self.token_endpoint = 'https://api.capitalone.com/oauth2/token'
self.deposits_endpoint = 'https://api.capitalone.com/deposits/'
if sandbox:
self.token_endpoint = 'https://api-sandbox.capitalone.com/oauth2/token'
self.deposits_endpoint = 'https://api-sandbox.capitalone.com/deposits/'
self._access_token = None
self.expires_at = 0
# construct handlers
self.requests_handler = requests_handler
self.response_handler = depositsHandler(usage_client)
# retrieve access token
self.retrieve_details = retrieve_details
self.products = None
if retrieve_details:
self.access_token()
self._get_products()
def requestDeconstructor(request_object, request_schema):
request_details = {}
request_model = jsonModel(request_schema)
try:
request_details = request_object.get_json(silent=True)
except:
return request_details
if not request_details:
return request_details
request_details = request_model.ingest(**request_details)
return request_details
def requestDeconstructor(request_object, request_schema):
request_details = {}
request_model = jsonModel(request_schema)
try:
request_details = request_object.get_json(silent=True)
except:
return request_details
if not request_details:
return request_details
request_details = request_model.ingest(**request_details)
return request_details
def __init__(self, regex_schema, override=False):
'''
instantiates class with a regular expression dictionary
:param regex_schema: dictionary with regular expression name, pattern key-pairs
:param override: boolean to ignore value errors raised from regex name conflicts
'''
class_name = self.__class__.__name__
# construct class method validator
self.fields = jsonModel(self._class_methods)
# validate inputs
object_title = '%s.__init__(regex_schema={...})' % class_name
regex_schema = self.fields.validate(regex_schema, '.__init__.regex_schema', object_title)
# construct builtin list to differentiate custom methods
self.builtins = []
for item in self.__dir__():
self.builtins.append(item)
# construct a method for each regex pattern in input
for key, value in regex_schema.items():
if key in self.builtins:
if not override:
raise ValueError('\nRegex key %s is the name of a %s builtin method.' % (key, class_name))
else:
pro_char = re.compile('[^\w]')
first_char = re.compile('[a-zA-Z]')
method_name = pro_char.sub('_', key)
if first_char.match(method_name):
try:
getattr(self, method_name)
if not override:
raise ValueError('\nThe method for regex key %s will conflict with another key.' % key)
except:
pass
sub_methods = {
'pattern': re.compile(value),
'name': key
}
method_object = _method_constructor(sub_methods)
setattr(self, method_name, method_object)
elif not override:
raise ValueError('\Regex key %s must begin with a letter.' % key)
def ingest_environ(model_path=''):
# convert environment variables into json typed data
from os import environ, path
typed_dict = {}
environ_variables = dict(environ)
for key, value in environ_variables.items():
if value.lower() == 'true':
typed_dict[key] = True
elif value.lower() == 'false':
typed_dict[key] = False
elif value.lower() == 'null':
typed_dict[key] = None
elif value.lower() == 'none':
typed_dict[key] = None
else:
try:
try:
typed_dict[key] = int(value)
except:
typed_dict[key] = float(value)
except:
typed_dict[key] = value
# feed environment variables through model
if model_path:
from labpack.records.settings import load_settings
if not path.exists(model_path):
raise ValueError('%s is not a valid file path.' % model_path)
model_dict = load_settings(model_path)
from jsonmodel.validators import jsonModel
model_object = jsonModel(model_dict)
default_dict = model_object.ingest(**{})
for key in default_dict.keys():
if key.upper() in typed_dict:
valid_kwargs = {
'input_data': typed_dict[key.upper()],
'object_title': 'Environment variable %s' % key.upper(),
'path_to_root': '.%s' % key
}
default_dict[key] = model_object.validate(**valid_kwargs)
return default_dict
return typed_dict
def __init__(self, access_id, secret_key, region_name, owner_id, user_name, verbose=True, usage_client=None):
'''
a method for initializing the connection to AWS Polly
:param access_id: string with access_key_id from aws IAM user setup
:param secret_key: string with secret_access_key from aws IAM user setup
:param region_name: string with name of aws region
:param owner_id: string with aws account id
:param user_name: string with name of user access keys are assigned to
:param verbose: boolean to enable process messages
:param usage_client: callable object to track resource usage
'''
title = '%s.__init__' % self.__class__.__name__
# initialize model
from labpack import __module__
from jsonmodel.loader import jsonLoader
from jsonmodel.validators import jsonModel
class_fields = jsonLoader(__module__, 'speech/aws/polly-rules.json')
self.fields = jsonModel(class_fields)
# construct iam connection
from labpack.authentication.aws.iam import iamClient
self.iam = iamClient(access_id, secret_key, region_name, owner_id, user_name, verbose)
# construct polly client connection
client_kwargs = {
'service_name': 'polly',
'region_name': self.iam.region_name,
'aws_access_key_id': self.iam.access_id,
'aws_secret_access_key': self.iam.secret_key
}
self.connection = boto3.client(**client_kwargs)
self.verbose = verbose
self.usage_client = usage_client
# construct range of polly options
self.voice_ids = self.fields.components['.voice_id']['discrete_values']
self.output_formats = self.fields.components['.output_format']['discrete_values']
# construct pythonic conversion method
from labpack.parsing.conversion import camelcase_to_lowercase
self.ingest = camelcase_to_lowercase
def compile_map(folder_path, file_suffix='', json_model=False):
from os import path
from labpack.records.settings import load_settings
file_map = {}
file_list = compile_list(folder_path, file_suffix)
for file_path in file_list:
file_details = load_settings(file_path)
file_key = path.split(file_path)[1].replace(file_suffix, '')
if json_model:
from jsonmodel.validators import jsonModel
file_map[file_key] = jsonModel(file_details)
else:
file_map[file_key] = file_details
return file_map
def __init__(self, access_token, collection_name=''):
'''
a method to initialize the dropboxClient class
:param access_token: string with oauth2 access token for users account
'''
title = '%s.__init__' % self.__class__.__name__
# construct input validation model
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'access_token': access_token,
'collection_name': collection_name
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# workaround for module namespace conflict
from sys import path as sys_path
sys_path.append(sys_path.pop(0))
from dropbox import Dropbox
from dropbox.files import FileMetadata, WriteMode, DeleteArg
from dropbox.exceptions import ApiError
sys_path.insert(0, sys_path.pop())
# construct dropbox client
from labpack.compilers.objects import _method_constructor
self.dropbox = Dropbox(oauth2_access_token=access_token)
# construct dropbox objects
self.objects = _method_constructor({
'FileMetadata': FileMetadata,
'ApiError': ApiError,
'WriteMode': WriteMode,
'DeleteArg': DeleteArg
})
# construct collection name
self.collection_name = collection_name
def __init__(self, group_name, server_url='ml.internalpositioning.com', password=''):
'''
a method to initialize a findClient class object
:param group_name: string with name of group
:param server_url: string with url for FIND server
:param password: [optional] string with password to mosquitto server
# https://www.internalpositioning.com/api
'''
title = '%s.__init__' % self.__class__.__name__
# construct fields
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'group_name': group_name,
'server_url': server_url,
'password': password
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct class properties
self.server_url = server_url
self.endpoint = 'https://%s' % server_url
self.endpoint_public = 'http://%s' % server_url
self.group_name = group_name
self.password = password
self.positions = {}
self.locations = {}
# add regex patterns
import re
self.user_pattern = re.compile('location/(.*)$')
def __init__(self, api_key, email_key, account_domain, usage_client=None, requests_handler=None):
'''
initialization method for mailgun client class
:param api_key: string with api key provided by mailgun
:param email_key: string with email validation key provide by mailgun
:param account_domain: string with domain from which to send email
:param usage_client: callable that records usage data
:param requests_handler: callable that handles requests errors
'''
title = '%s.__init__' % self.__class__.__name__
# construct class field model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'api_key': api_key,
'email_key': email_key,
'account_domain': account_domain
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct client properties
self.api_endpoint = self.fields.schema['api_endpoint']
self.account_domain = account_domain
self.api_key = api_key
self.email_key = email_key
# construct handlers
self.service_handler = mailgunHandler(usage_client)
self.requests_handler = requests_handler
def __init__(self, service_username, service_password, requests_handler=None, magic_file=''):
title = '%s.__init__' % self.__class__.__name__
# construct class field model
from jsonmodel.validators import jsonModel
for key, value in self._class_fields['schema']['audio_extensions'].items():
self._class_fields['components']['.audio_mimetype']['discrete_values'].append(value['mimetype'])
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'service_username': service_username,
'service_password': service_password
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# define class properties
self.username = service_username
self.password = service_password
# construct watson client
self.client = SpeechToTextV1(username=self.username, password=self.password)
# construct handlers
self.requests_handler = requests_handler
# construct magic format checker
self.magic = None
if magic_file:
from labpack.parsing.magic import labMagic
self.magic = labMagic(magic_file)
# import validate extension
from labpack.parsing.regex import validate_extension
self._validate_extension = validate_extension
def __init__(self,
account_email,
account_password,
app_subdomain,
verbose=False):
''' a method to initialize the herokuClient class '''
title = '%s.__init__' % self.__class__.__name__
# construct fields model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'account_email': account_email,
'account_password': account_password,
'app_subdomain': app_subdomain
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
self.email = account_email
self.password = account_password
self.subdomain = app_subdomain
# construct class properties
self.verbose = verbose
# construct localhost
from labpack.platforms.localhost import localhostClient
self.localhost = localhostClient()
# validate installation
self._validate_install()
# validate access
self._validate_access()
def __init__(self, usage_client=None):
'''
initialization method for deposits handler class
:param usage_client: callable that records usage data
'''
# construct class field model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# construct initial methods
self.rate_limits = self.fields.schema['rate_limits']
self.usage_client = usage_client
# construct error map
self.error_map = {
429: "The request has been rejected because of rate limiting -- you've sent too many requests in a given amount of time.",
500: "General server error.",
502: "Internal connection failure.",
503: "The server is unavailable due to heavy traffic or maintenance."
}
def __init__(self, account_email, auth_token, verbose=True):
''' a method to initialize the herokuClient class '''
title = '%s.__init__' % self.__class__.__name__
# initialize super
super(herokuClient, self).__init__(verbose=verbose)
# construct fields model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'account_email': account_email,
'auth_token': auth_token
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct properties
self.email = account_email
self.token = auth_token
self.subdomain = ''
self.apps = []
# construct localhost
from labpack.platforms.localhost import localhostClient
self.localhost = localhostClient()
# validate installation
self._validate_install()
# validate access
self._validate_login()
def __init__(self, bot_id, access_token, requests_handler=None):
''' initialization method for moves client class
:param bot_id: integer with telegram id number for bot
:param access_token: string with access token for bot provided by telegram botfather
:param requests_handler: callable that handles requests errors
'''
# construct class field model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# construct client attributes
object_title = '%s.__init__(bot_id=%s)' % (self.__class__.__name__, str(bot_id))
self.bot_id = self.fields.validate(bot_id, '.bot_id', object_title)
object_title = '%s.__init__(access_token=%s)' % (self.__class__.__name__, str(access_token))
self.access_token = self.fields.validate(access_token, '.access_token', object_title)
self.api_endpoint = '%s%s:%s' % (self.fields.schema['api_endpoint'], self.bot_id, self.access_token)
self.file_endpoint = '%s%s:%s/' % (self.fields.schema['file_endpoint'], self.bot_id, self.access_token)
# construct handlers
self.requests_handler = requests_handler
self.telegram_handler = telegramBotHandler()
def start(service_list, verbose=True, virtualbox='default'):
title = 'start'
# validate inputs
if isinstance(service_list, str):
if service_list:
service_list = [service_list]
input_fields = {
'service_list': service_list,
'verbose': verbose,
'virtualbox': virtualbox
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
fields_model.validate(value, '.%s' % key, object_title)
# validate installation of docker
from pocketlab.methods.docker import dockerClient
docker_client = dockerClient(virtualbox_name=virtualbox, verbose=verbose)
# verbosity
if verbose:
print('Checking service configurations...', end='', flush=True)
# construct list of paths to services
from pocketlab.methods.service import retrieve_services
start_list, msg_insert = retrieve_services(service_list)
# construct lab list
lab_list = []
# validate lab files
from os import path
from pocketlab.methods.validation import validate_lab
from pocketlab import __module__
from jsonmodel.loader import jsonLoader
from jsonmodel.validators import jsonModel
lab_model = jsonModel(jsonLoader(__module__, 'models/lab-config.json'))
for details in start_list:
file_path = path.join(details['path'], 'lab.yaml')
service_name = details['name']
lab_details = validate_lab(lab_model, file_path, service_name)
details['config'] = lab_details
lab_list.append(details)
# retrieve list of docker images
# validate lab config image exists
# retrieve list of docker containers
# validate lab config container doesn't already exist
# TODO resolve name conflicts on deploy updating
# validate mount paths exist
if verbose:
print('.', end='', flush=True)
# end validations
if verbose:
print(' done.')
# retrieve system ip
system_ip = docker_client.ip()
# instantiate containers
exit_msg = ''
port_list = []
for service in lab_list:
service_config = service['config']
service_name = service['name']
service_root = service['path']
# construct environment variables
container_envvar = { 'SYSTEM_IP_ADDRESS': system_ip }
if service_config['docker_environment_variables']:
for key, value in service_config['docker_environment_variables'].items():
container_envvar[key.upper()] = '"%s"' % value
# construct sys command
container_name = service_config['docker_container_alias']
sys_command = 'docker run --name %s' % container_name
for key, value in container_envvar.items():
sys_command += ' -e %s=%s' % (key, value)
if service_config['docker_mount_volumes']:
for key, value in service_config['docker_mount_volumes'].items():
sys_command += ' -v "%s":"%s"' % (key, value)
sys_command += ' -it -d'
port_count = 0
port_value = ''
if service_config['docker_port_mapping']:
for key, value in service_config['docker_port_mapping'].items():
port_value = value
if value in port_list:
from copy import deepcopy
port_value = deepcopy(value)
for i in range(1000):
port_value += 1
if not port_value in port_list:
break
port_list.append(port_value)
port_count += 1
sys_command += ' -p %s:%s' % (str(port_value), key)
sys_command += ' %s' % service_config['docker_image_name']
# determine port message
port_msg = ''
if port_count == 1:
port_msg = ' at %s:%s' % (system_ip, port_value)
# run command
from subprocess import check_output
docker_response = check_output(sys_command).decode('utf-8')
service_msg = 'Container "%s" started%s' % (container_name, port_msg)
if len(lab_list) > 1:
if verbose:
print(service_msg)
else:
exit_msg = service_msg
# TODO consider ROLLBACK options
if len(lab_list) > 1:
exit_msg = 'Finished starting %s' % msg_insert
return exit_msg
# # import dependencies
# from os import path
# from copy import deepcopy
# from pocketlab.importers.config_file import configFile
# from pocketlab.clients.localhost_client import localhostClient
# from pocketlab.clients.docker_session import dockerSession
# from pocketlab.validators.config_model import configModel
# from pocketlab.validators.absolute_path import absolutePath
# from pocketlab.validators.available_image import availableImage
# from pocketlab.compilers.docker_run import dockerRun
#
# # ingest verbose options
# verbose = kwargs['verbose']
#
# # determine system properties
# localhost = localhostClient()
#
# # ingest & validate component file
# component_file = kwargs['componentFile']
# comp_details = configFile(component_file, kwargs)
# comp_details = configModel(comp_details, 'rules/lab-component-model.json', kwargs, 'component settings')
#
# # determine component root from component file
# root_path, file_name = path.split(path.abspath(component_file))
#
# # construct path details to mounted volumes
# mounted_volumes = {}
# for volume in comp_details['mounted_volumes']:
# host_path = absolutePath(volume, root_path, kwargs, 'mounted volume')
# system_path = host_path.replace('\\','/').replace('C:','//c')
# absolute_path = deepcopy(host_path)
# docker_path = absolute_path.replace(root_path,'')
# container_path = docker_path.replace('\\','/')
# mounted_volumes[system_path] = container_path
#
# # ingest & validate virtualbox property
# vbox_name = kwargs['virtualbox']
# if not localhost.os in ('Windows','Mac'):
# vbox_name = ''
#
# # check for docker installation
# docker_session = dockerSession(kwargs, vbox_name)
#
# # check that docker image is available locally
# image_list = docker_session.images()
# availableImage(comp_details['docker_image'], comp_details['image_tag'], image_list, kwargs)
#
# # retrieve list of active container aliases & busy ports
# container_list = docker_session.ps()
# busy_ports = []
# active_containers = []
# for container in container_list:
# active_containers.append(container['NAMES'])
# container_settings = docker_session.inspect(container_alias=container['NAMES'])
# container_synopsis = docker_session.synopsis(container_settings)
# if container_synopsis['mapped_ports']:
# for key in container_synopsis['mapped_ports'].keys():
# busy_ports.append(key)
#
# # check that alias name is available
# if comp_details['container_alias'] in active_containers:
# from pocketlab.exceptions.lab_exception import labException
# header_list = [ 'NAMES', 'STATUS', 'IMAGE', 'PORTS']
# error = {
# 'kwargs': kwargs,
# 'message': 'Container "%s" already in use. Containers currently active:' % comp_details['container_alias'],
# 'tprint': { 'headers': header_list, 'rows': container_list },
# 'error_value': comp_details['container_alias'],
# 'failed_test': 'unavailable_resource'
# }
# raise labException(**error)
#
# # construct port mappings for mapped ports
# mapped_ports = {}
# for port in comp_details['exposed_ports']:
# open_port = int(deepcopy(port))
# while str(open_port) in busy_ports:
# open_port += 1
# mapped_ports[str(open_port)] = str(port)
#
# # add system_ip to injected variables
# system_ip = docker_session.ip()
# injected_variables = {
# 'SYSTEM_LOCALHOST': system_ip
# }
# for key, value in comp_details['injected_variables'].items():
# injected_variables[key] = value
#
# # compile docker run script from settings
# run_details = {
# 'name': comp_details['container_alias'],
# 'injected_variables': injected_variables,
# 'mounted_volumes': mounted_volumes,
# 'mapped_ports': mapped_ports,
# 'docker_image': comp_details['docker_image'],
# 'image_tag': comp_details['image_tag'],
# 'run_command': comp_details['run_command']
# }
# run_script = dockerRun(run_details)
#
# # start container
# container_id = docker_session.run(run_script)
# if verbose:
# start_text = 'Sweet! Container "%s" started' % comp_details['container_alias']
# if run_details['mapped_ports']:
# start_text += ' on port'
# if len(run_details['mapped_ports'].keys()) > 1:
# start_text += 's'
# previous_port = False
# for key in run_details['mapped_ports'].keys():
# if previous_port:
# start_text += ','
# start_text += ' %s:%s' % (system_ip, key)
# previous_port = True
# start_text += '.'
# print(start_text)
#
# container_details = {
# 'mapped_ports': mapped_ports,
# 'container_alias': comp_details['container_alias'],
# 'container_id': container_id
# }
#
# return container_details
def stop(service_list, verbose=True, virtualbox='default'):
title = 'stop'
# validate inputs
if isinstance(service_list, str):
if service_list:
service_list = [service_list]
input_fields = {
'service_list': service_list,
'verbose': verbose,
'virtualbox': virtualbox
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
fields_model.validate(value, '.%s' % key, object_title)
# validate installation of docker
from pocketlab.methods.docker import dockerClient
docker_client = dockerClient(virtualbox_name=virtualbox, verbose=verbose)
# verbosity
if verbose:
print('Checking service configurations...', end='', flush=True)
# construct list of paths to services
from pocketlab.methods.service import retrieve_services
stop_list, msg_insert = retrieve_services(service_list)
# construct lab list
lab_list = []
# validate lab files
from os import path
from pocketlab.methods.validation import validate_lab
from pocketlab import __module__
from jsonmodel.loader import jsonLoader
from jsonmodel.validators import jsonModel
lab_model = jsonModel(jsonLoader(__module__, 'models/lab-config.json'))
for details in stop_list:
file_path = path.join(details['path'], 'lab.yaml')
service_name = details['name']
lab_details = validate_lab(lab_model, file_path, service_name)
details['config'] = lab_details
lab_list.append(details)
# retrieve list of docker containers
# validate lab config container exists
# TODO resolve name discovery when namespace transmutes
if verbose:
print('.', end='', flush=True)
# end validations
if verbose:
print(' done.')
# stop containers
exit_msg = ''
for service in lab_list:
container_alias = service['config']['docker_container_alias']
docker_status = docker_client.rm(container_alias)
if docker_status == container_alias:
service_msg = 'Container "%s" terminated.' % docker_status
if len(lab_list) > 1:
if verbose:
print(service_msg)
else:
exit_msg = service_msg
else:
raise ValueError(docker_status)
if len(lab_list) > 1:
exit_msg = 'Finished terminating %s' % msg_insert
return exit_msg
def retrieve_oauth2_configs(folder_path=''):
''' a method to retrieve oauth2 configuration details from files or envvar '''
# define oauth2 model
oauth2_fields = {
"schema": {
"oauth2_app_name": "My App",
"oauth2_developer_name": "Collective Acuity",
"oauth2_service_name": "moves",
"oauth2_auth_endpoint": "https://api.moves-app.com/oauth/v1/authorize",
"oauth2_token_endpoint": "https://api.moves-app.com/oauth/v1/access_token",
"oauth2_client_id": "ABC-DEF1234ghijkl-567MNOPQRST890uvwxyz",
"oauth2_client_secret": "abcdefgh01234456789_IJKLMNOPQrstuv-wxyz",
"oauth2_redirect_uri": "https://collectiveacuity.com/authorize/moves",
"oauth2_service_scope": "activity location",
"oauth2_response_type": "code",
"oauth2_request_mimetype": "",
"oauth2_service_logo": "https://pbs.twimg.com/profile_images/3/d_400x400.png",
"oauth2_service_description": "",
"oauth2_service_setup": 0.0
}
}
# retrieve keys, value pairs from config files in cred folder
if folder_path:
envvar_details = {}
import os
from labpack.records.settings import load_settings
file_list = []
for suffix in ['.yaml', '.yml', '.json']:
for file_name in os.listdir(folder_path):
file_path = os.path.join(folder_path, file_name)
if os.path.isfile(file_path):
if file_name.find(suffix) > -1:
file_list.append(file_path)
for file_path in file_list:
file_details = load_settings(file_path)
envvar_details.update(**file_details)
# or ingest environmental variables
else:
from labpack.records.settings import ingest_environ
envvar_details = ingest_environ()
# map oauth2 variables
import re
oauth2_map = {}
for key in oauth2_fields['schema'].keys():
key_pattern = '%s$' % key[6:]
key_regex = re.compile(key_pattern)
for k, v in envvar_details.items():
if key_regex.findall(k.lower()):
service_name = key_regex.sub('',k.lower())
if not service_name in oauth2_map.keys():
oauth2_map[service_name] = {}
oauth2_map[service_name][key] = v
# ingest models
from jsonmodel.validators import jsonModel
oauth2_model = jsonModel(oauth2_fields)
oauth2_services = {}
for key, value in oauth2_map.items():
valid_oauth2 = {}
try:
valid_oauth2 = oauth2_model.validate(value)
except:
pass
if valid_oauth2:
oauth2_services[key] = valid_oauth2
return oauth2_services
from jsonmodel.loader import jsonLoader
from pocketlab.methods.config import compile_yaml
config_schema = jsonLoader(__module__, value['schema_path'])
config_text = compile_yaml(config_schema)
with open(config_path, 'wt') as f:
f.write(config_text)
f.close()
_printer(config_path)
# add readme file
readme_path = 'README.md'
if not path.exists(readme_path):
from pocketlab.methods.config import construct_readme
readme_text = construct_readme(vcs_service=vcs_service)
with open(readme_path, 'wt', encoding='utf-8') as f:
f.write(readme_text)
f.close()
_printer(readme_path)
exit_msg = 'Lab framework setup in current directory.'
return exit_msg
if __name__ == "__main__":
from labpack.records.settings import load_settings
from jsonmodel.validators import jsonModel
config_path = '../models/lab-config.json'
config_model = jsonModel(load_settings(config_path))
print(config_model.ingest())
def compile_settings(model_path, file_path, ignore_errors=False):
''' a method to compile configuration values from different sources
NOTE: method searches the environment variables, a local
configuration path and the default values for a jsonmodel
object for valid configuration values. if an environmental
variable or key inside a local config file matches the key
for a configuration setting declared in the jsonmodel schema,
its value will be added to the configuration file as long
as the value is model valid. SEE jsonmodel module.
NOTE: the order of assignment:
first: environment variable
second: configuration file
third: default value
fourth: empty value
NOTE: method is guaranteed to produce a full set of top-level keys
:param model_path: string with path to jsonmodel valid model data
:param file_path: string with path to local configuration file
:param ignore_errors: [optional] boolean to ignore any invalid values
:return: dictionary with settings
'''
# construct configuration model and default details
from jsonmodel.validators import jsonModel
config_model = jsonModel(load_settings(model_path))
default_details = config_model.ingest(**{})
# retrieve environmental variables and file details
environ_details = ingest_environ()
try:
file_details = load_settings(file_path)
except:
file_details = {}
# construct config details from (first) envvar, (second) file, (third) default
config_details = {}
for key in default_details.keys():
test_file = True
test_default = True
if key.upper() in environ_details.keys():
test_file = False
test_default = False
try:
config_details[key] = config_model.validate(environ_details[key.upper()], '.%s' % key)
except:
if ignore_errors:
test_file = True
test_default = True
else:
raise
if key in file_details.keys() and test_file:
test_default = False
try:
config_details[key] = config_model.validate(file_details[key], '.%s' % key)
except:
if ignore_errors:
test_default = True
else:
raise
if test_default:
config_details[key] = default_details[key]
return config_details
def compile_yaml(config_schema, yaml_path=''):
'''
a method to compile a yaml file with top-level comments from a json model
:param config_schema: dictionary with json model schema architecture
:param yaml_path: [optional] string with path to user yaml file
:return: string with yaml formatted text
'''
# construct config model
from jsonmodel.validators import jsonModel
config_model = jsonModel(config_schema)
# construct order dict
config_list = []
config_details = config_model.ingest()
for key, value in config_details.items():
details = {
'key': key,
'value': value,
'position': 0,
'comments': ''
}
comp_key = '.%s' % key
if comp_key in config_model.keyMap.keys():
if 'field_metadata' in config_model.keyMap[comp_key].keys():
metadata = config_model.keyMap[comp_key]['field_metadata']
if 'position' in metadata.keys():
if isinstance(metadata['position'], int):
details['position'] = metadata['position']
if 'comments' in metadata.keys():
if isinstance(metadata['comments'], str):
details['comments'] = metadata['comments']
config_list.append(details)
config_list.sort(key=lambda k: k['position'])
# construct config text
config_text = ''
if 'comments' in config_model.metadata.keys():
if isinstance(config_model.metadata['comments'], str):
comment_lines = config_model.metadata['comments'].splitlines()
for comment in comment_lines:
config_text += '# %s\n' % comment
for item in config_list:
comment_stub = '\n'
if item['comments']:
comment_stub = ' # %s\n' % item['comments']
try:
float(item['key'])
item_key = "'%s'" % item['key']
except:
item_key = item['key']
if isinstance(item['value'], dict):
line_text = '%s:%s' % (item_key, comment_stub)
for key, value in item['value'].items():
try:
float(key)
key_text = "'%s'" % key
except:
key_text = key
value_text = value
if isinstance(value, str):
try:
float(value)
value_text = "'%s'" % value
except:
pass
line_text += ' %s: %s\n' % (key_text, value_text)
elif isinstance(item['value'], list):
line_text = '%s:%s' % (item_key, comment_stub)
for i in range(len(item['value'])):
value_text = item['value'][i]
if isinstance(value_text, str):
try:
float(value_text)
value_text = "'%s'" % value_text
except:
pass
line_text += ' - %s\n' % value_text
else:
line_text = '%s: %s%s' % (item_key, str(item['value']), comment_stub)
config_text += line_text
# update user config
if yaml_path:
from os import path
if not path.exists(yaml_path):
raise ValueError('%s is not a valid path.')
import ruamel.yaml
user_text = open(yaml_path).read()
user_code = ruamel.yaml.load(user_text, ruamel.yaml.RoundTripLoader)
user_len = len(user_code.keys())
count = 0
for item in config_list:
if item['key'] not in user_code.keys():
insert_row = user_len + count
insert_kwargs = {
'pos': insert_row,
'key': item['key'],
'value': item['value']
}
if item['comments']:
insert_kwargs['comment'] = item['comments']
user_code.insert(**insert_kwargs)
count += 1
config_text = ruamel.yaml.dump(user_code, Dumper=ruamel.yaml.RoundTripDumper)
return config_text
def __init__(self, bucket_name, database_url, document_schema=None, verbose=False, public=False, configs=None):
''' the initialization method for syncGatewayAdmin class '''
# https://developer.couchbase.com/documentation/mobile/1.5/guides/sync-gateway/config-properties/index.html
# https://developer.couchbase.com/documentation/mobile/current/guides/sync-gateway/sync-function-api-guide/index.html
title = '%s.__init__' % self.__class__.__name__
# import default sync function
from os import path
from importlib.util import find_spec
module_path = find_spec(__module__).submodule_search_locations[0]
sync_path = path.join(module_path, 'databases/models/sync_function.js')
sync_text = open(sync_path).read()
sync_text = self._update_js(sync_text, document_schema)
# construct fields
self._class_fields['components']['.configs.sync']['default_value'] = sync_text
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'bucket_name': bucket_name,
'database_url': database_url,
'document_schema': document_schema,
'configs': configs
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# test connection to db
self.admin_access = True
try:
response = requests.get(database_url)
response = response.json()
if not 'ADMIN' in response.keys():
self.admin_access = False
except:
raise Exception('%s(database_url="%s") is not a valid couchbase url.' % (title, database_url))
# construct class properties
from os import path
self.bucket_name = bucket_name
self.database_url = database_url
self.bucket_url = path.join(database_url, bucket_name)
# construct verbose method
self.printer_on = True
def _printer(msg, flush=False):
if verbose and self.printer_on:
if flush:
print(msg, end='', flush=True)
else:
print(msg)
self.printer = _printer
# construct document model
from jsonmodel.validators import jsonModel
self.public = public
self.model = None
if document_schema:
self.model = jsonModel(document_schema)
elif not self.public:
self.model = jsonModel({'schema': { 'uid': 'abc012XYZ789' }, 'components': {'.': { 'extra_fields': True}}})
if self.model:
if not 'uid' in self.model.schema.keys():
document_schema['schema']['uid'] = 'abc012XYZ789'
if not self.model.components:
document_schema['components'] = { '.': { 'extra_fields': True } }
elif not '.' in self.model.components.keys():
document_schema['components']['.'] = { 'extra_fields': True }
elif not 'extra_fields' in self.model.components['.'].keys():
document_schema['components']['.']['extra_fields'] = True
self.model = jsonModel(document_schema)
# construct configs
default_fields = self.fields.ingest(**{})
self.configs = default_fields['configs']
self.configs['bucket'] = self.bucket_name
self.configs['name'] = self.bucket_name
if self.public:
from jsonmodel.loader import jsonLoader
read_only_configs = jsonLoader(__module__, 'databases/models/sync-gateway-public.json')
for key, value in read_only_configs.items():
self.configs[key] = value
if configs:
for key, value in configs.items():
self.configs[key] = value
# construct lab record generator
from labpack.records.id import labID
self.labID = labID
# create db (update config) if none exists
self._update_bucket()
# construct sql tables
from labpack.databases.sql import sqlClient
sql_tables = {}
for key, value in object_map.items():
table_name = key.replace('-', '_')
sql_kwargs = {
'table_name': table_name,
'database_url': 'sqlite:///../data/records.db',
'record_schema': value
}
sql_tables[table_name] = sqlClient(**sql_kwargs)
# construct data object models
api_model = load_settings('models/api.json')
telemetry_model = jsonModel(load_settings('models/telemetry-post.json'))
# construct email client
from labpack.email.mailgun import mailgunClient
from labpack.handlers.requests import handle_requests
mailgun_cred = load_settings('../cred/mailgun.yaml')
mailgun_kwargs = {
'api_key': mailgun_cred['mailgun_api_key'],
'email_key': mailgun_cred['mailgun_email_key'],
'account_domain': mailgun_cred['mailgun_spf_route'],
'requests_handler': handle_requests
}
email_client = mailgunClient(**mailgun_kwargs)
if __name__ == "__main__":
class resourceModel(object):
def __init__(self, json_model):
if not isinstance(json_model, jsonModel):
raise TypeError('resourceModel input must be a jsonModel object.')
self.model = json_model
self.request = requestModel(self.model)
self.database = databaseModel(self.model)
# import json object files
operationFile = json.loads(open('models/operation-model.json').read())
catchFile = json.loads(open('models/catch-model.json').read())
specimenFile = json.loads(open('models/specimen-model.json').read())
# construct object models from files
operationModel = jsonModel(operationFile)
catchModel = jsonModel(catchFile)
specimenModel = jsonModel(specimenFile)
# construct resource objects from object models
operationResource = resourceModel(operationModel)
catchResource = resourceModel(catchModel)
specimenResource = resourceModel(specimenModel)
if __name__ == '__main__':
print(operationResource.request.model.schema)
raise TypeError('databaseModel input must be a jsonModel object.')
self.model = json_model
class resourceModel(object):
def __init__(self, json_model):
if not isinstance(json_model, jsonModel):
raise TypeError('resourceModel input must be a jsonModel object.')
self.model = json_model
self.request = requestModel(self.model)
self.database = databaseModel(self.model)
# import json object files
operationFile = json.loads(open('models/operation-model.json').read())
catchFile = json.loads(open('models/catch-model.json').read())
specimenFile = json.loads(open('models/specimen-model.json').read())
# construct object models from files
operationModel = jsonModel(operationFile)
catchModel = jsonModel(catchFile)
specimenModel = jsonModel(specimenFile)
# construct resource objects from object models
operationResource = resourceModel(operationModel)
catchResource = resourceModel(catchModel)
specimenResource = resourceModel(specimenModel)
if __name__ == '__main__':
print(operationResource.request.model.schema)
def __init__(self):
''' a method to initialize a client class to interact with the localhost '''
# construct class field input validation property
self.fields = jsonModel(self._class_fields)
# retrieve operating system from localhost
self.os = osClient()
# TODO: determine file system and parameters
# TODO: request latest info from
# https://en.wikipedia.org/wiki/Comparison_of_file_systems#Limits
# retrieve IP from system
self.os.nodename = gethostname()
self.ip = gethostbyname(self.os.nodename)
# retrieve environment variables from system
self.environ = dict(os.environ.items())
# retrieve path to user home
self.home = ''
if self.os.sysname == 'Windows':
env_username = os.environ.get('USERNAME')
from re import compile
xp_pattern = compile('^C:\\Documents and Settings')
app_data = ''
if os.environ.get('APPDATA'):
app_data = os.environ.get('APPDATA')
if xp_pattern.findall(app_data):
self.home = 'C:\\Documents and Settings\\%s' % env_username
else:
self.home = 'C:\\Users\\%s' % env_username
elif self.os.sysname in ('Linux', 'FreeBSD', 'Solaris', 'Darwin'):
self.home = os.path.expanduser('~')
# retrieve path to shell configs
self.bash_config = ''
self.sh_config = ''
if self.os.sysname == 'Windows':
bash_config = '.bash_profile'
sh_config = ''
else:
bash_config = '.bashrc'
sh_config = '.cshrc'
if bash_config:
self.bash_config = os.path.join(self.home, bash_config)
if sh_config:
self.sh_config = os.path.join(self.home, sh_config)
# TODO check different terminal protocols
# construct file record model property
file_model = {
'schema': {
'file_name': 'test.json',
'file_path': '/home/user/.config/collective-acuity-labpack/user-data/test.json',
'file_size': 678,
'create_date': 1474509314.419702,
'update_date': 1474509314.419702,
'access_date': 1474509314.419702
},
'components': {
'.file_size': {
'integer_data': True
}
}
}
self.file_model = jsonModel(file_model)
def unitTests(self, valid_input, valid_query):
# print(self.keyMap)
# test model fields
assert isinstance(self.title, str)
assert isinstance(self.description, str)
assert isinstance(self.url, str)
assert isinstance(self.metadata, dict)
assert isinstance(self.maxSize, int)
# test declarative fields in model keyMap
assert self.keyMap["."]["value_datatype"]
assert self.keyMap[".userID"]["required_field"]
assert self.keyMap[".address.region"]["declared_value"]
assert self.keyMap[".rating"]["default_value"]
assert self.keyMap[".emoticon"]["byte_data"]
assert self.keyMap[".rating"]["integer_data"]
assert self.keyMap[".userID"]["min_length"]
assert self.keyMap[".comments[0]"]["max_length"]
assert self.keyMap[".rating"]["min_value"]
assert self.keyMap[".rating"]["max_value"]
assert self.keyMap[".comments"]["min_size"]
assert self.keyMap[".comments"]["max_size"]
assert self.keyMap[".comments"]["unique_values"]
assert self.keyMap[".userID"]["must_not_contain"]
assert self.keyMap[".comments[0]"]["must_contain"]
assert self.keyMap[".address.region"]["contains_either"]
assert self.keyMap[".address.country_code"]["discrete_values"]
assert self.keyMap[".emoticon"]["example_values"]
assert self.keyMap[".address.region"]["field_title"]
assert self.keyMap[".userID"]["field_description"]
assert self.keyMap[".emoticon"]["field_metadata"]
# TODO: "identical_to": ".similar_string",
# TODO: "lambda_function": "",
# TODO: "validation_url": "",
# test declared values for empty values
assert isinstance(self.keyMap[".address.country_code"]["declared_value"], int)
assert isinstance(self.keyMap[".address.postal_code"]["declared_value"], str)
# test integers in key name exception
try:
test_schema = {"schema": {0: "value"}}
jsonModel(test_schema)
except ModelValidationError as err:
assert str(err).find("Model declaration is invalid")
# test validation with empty path to root
v_input = deepcopy(valid_input)
assert self.validate(v_input)
# test validation with dot-path to root
v_input = deepcopy(valid_input)
assert self.validate(v_input, ".")
# test individual component validation
v_input = deepcopy(valid_input)
assert self.validate(v_input["datetime"], ".datetime") == v_input["datetime"]
v_input = deepcopy(valid_input)
assert self.validate(v_input["userID"], ".userID") == v_input["userID"]
v_input = deepcopy(valid_input)
assert not self.validate(v_input["active"], ".active")
v_input = deepcopy(valid_input)
assert self.validate(v_input["comments"], ".comments") == v_input["comments"]
v_input = deepcopy(valid_input)
assert self.validate(v_input["address"], ".address") == v_input["address"]
# test non-existent path to root exception
v_input = deepcopy(valid_input)
try:
self.validate(v_input, ".not_a_path")
except ModelValidationError as err:
assert str(err).find("Model declaration is invalid")
# test path to root not a string
v_input = deepcopy(valid_input)
try:
self.validate(v_input, [".datetime"])
except ModelValidationError as err:
assert str(err).find("Model declaration is invalid")
# test invalid input type
invalid_list = []
try:
self.validate(invalid_list)
except InputValidationError as err:
assert err.error["model_schema"]
assert err.error["failed_test"] == "value_datatype"
# test object title in error message
try:
self.validate(invalid_list, object_title="List input")
except InputValidationError as err:
assert err.error["object_title"]
assert str(err).find("input is invalid.")
assert str(err).find("Value []")
# test non-json valid object exception
invalid_object = jsonModel({"schema": {"test": "object"}})
try:
self.validate(invalid_object, object_title="jsonModel input")
except InputValidationError as err:
assert str(err).find("Value jsonModel")
# test json structure of error message
try:
self.validate(invalid_list)
except InputValidationError as err:
assert json.dumps(err.error)
try:
self.validate(invalid_object)
except InputValidationError as err:
assert json.dumps(err.error)
# test invalid input data type
try:
self.validate("1449179763.312077", ".datetime")
except InputValidationError as err:
assert err.error["failed_test"] == "value_datatype"
assert json.dumps(err.error)
# test non-json valid input datatype
try:
self.validate(invalid_object, ".comments[0]")
except InputValidationError as err:
assert err.error["failed_test"] == "value_datatype"
assert json.dumps(err.error)
# test key_datatype exception
integer_keyname_error = deepcopy(valid_input)
integer_keyname_error[2] = "integer key name"
try:
self.validate(integer_keyname_error)
except InputValidationError as err:
assert err.error["error_value"] == 2
# test extra_fields exception
extra_key_input = deepcopy(valid_input)
extra_key_input["extraKey"] = "string"
try:
self.validate(extra_key_input)
except InputValidationError as err:
assert err.error["failed_test"] == "extra_fields"
assert not err.error["object_title"]
# test required_field exception
missing_key_input = deepcopy(valid_input)
del missing_key_input["active"]
try:
self.validate(missing_key_input, object_title="Required field exception")
except InputValidationError as err:
assert err.error["failed_test"] == "required_field"
assert err.error["object_title"]
# test required_field false in dictionaries
optional_key = deepcopy(valid_input)
del optional_key["comments"]
assert not "comments" in self.validate(optional_key).keys()
# test default_value insertion
default_rating = deepcopy(valid_input)
new_default_rating = self.validate(default_rating)
assert new_default_rating["rating"] == 5
# test min_size exception
short_list = deepcopy(valid_input)
short_list["comments"] = []
try:
self.validate(short_list)
except InputValidationError as err:
assert err.error["failed_test"] == "min_size"
assert not err.error["object_title"]
# test max_size exception
long_list = deepcopy(valid_input)
long_list["comments"].append("pewter")
try:
self.validate(long_list, object_title="Max size exception")
except InputValidationError as err:
assert err.error["failed_test"] == "max_size"
assert err.error["object_title"]
# test value_datatype exception
mixed_list = deepcopy(valid_input)
mixed_list["comments"][1] = 100
try:
self.validate(mixed_list)
except InputValidationError as err:
assert err.error["failed_test"] == "value_datatype"
# test unique_values exception
duplicate_list = deepcopy(valid_input)
duplicate_list["comments"][2] = "gold"
try:
self.validate(duplicate_list)
except InputValidationError as err:
assert err.error["failed_test"] == "unique_values"
# test integer_data exception
integers_only = deepcopy(valid_input)
integers_only["rating"] = 3.5
try:
self.validate(integers_only, object_title="Integer data exception")
except InputValidationError as err:
assert err.error["failed_test"] == "integer_data"
assert err.error["object_title"]
# test min_value exception
min_number = deepcopy(valid_input)
min_number["rating"] = 0
try:
self.validate(min_number)
except InputValidationError as err:
assert err.error["failed_test"] == "min_value"
assert not err.error["object_title"]
# test max_value exception
max_number = deepcopy(valid_input)
max_number["rating"] = 11
try:
self.validate(max_number)
except InputValidationError as err:
assert err.error["failed_test"] == "max_value"
# test greater_than exception for numbers
greater_number = deepcopy(valid_input)
greater_number["datetime"] = 0.1
try:
self.validate(greater_number)
except InputValidationError as err:
assert err.error["failed_test"] == "greater_than"
# test less_than exception for numbers
less_number = deepcopy(valid_input)
less_number["datetime"] = 2000000000.1
try:
self.validate(less_number)
except InputValidationError as err:
assert err.error["failed_test"] == "less_than"
# test min_value for strings exception
low_string = deepcopy(valid_input)
low_string["userID"] = "0000000000000"
try:
self.validate(low_string)
except InputValidationError as err:
assert err.error["failed_test"] == "min_value"
# test max_value for strings exception
high_string = deepcopy(valid_input)
high_string["userID"] = "zzzzzzzzzzzzz"
try:
self.validate(high_string)
except InputValidationError as err:
assert err.error["failed_test"] == "max_value"
# test greater_than exception for strings
greater_string = deepcopy(valid_input)
greater_string["address"]["region"] = "AA"
try:
self.validate(greater_string, object_title="Greater than exception")
except InputValidationError as err:
assert err.error["failed_test"] == "greater_than"
assert err.error["object_title"]
# test less_than exception for strings
less_string = deepcopy(valid_input)
less_string["address"]["region"] = "Zzzzzzzzzzzzzzzzzzzzzzzz"
try:
self.validate(less_string)
except InputValidationError as err:
assert err.error["failed_test"] == "less_than"
assert not err.error["object_title"]
# test excluded_values for strings exception
excluded_string = deepcopy(valid_input)
excluded_string["emoticon"] = "c2Fk"
try:
self.validate(excluded_string)
except InputValidationError as err:
assert err.error["failed_test"] == "excluded_values"
# test excluded_values for strings exception
excluded_number = deepcopy(valid_input)
excluded_number["rating"] = 7
try:
self.validate(excluded_number)
except InputValidationError as err:
assert err.error["failed_test"] == "excluded_values"
# test discrete_values exception
discrete_string = deepcopy(valid_input)
discrete_string["address"]["city"] = "Boston"
try:
self.validate(discrete_string)
except InputValidationError as err:
assert err.error["failed_test"] == "discrete_values"
# test discrete_values exception
discrete_number = deepcopy(valid_input)
discrete_number["address"]["country_code"] = 20
try:
self.validate(discrete_number)
except InputValidationError as err:
assert err.error["failed_test"] == "discrete_values"
# test byte_data exception
byte_string = deepcopy(valid_input)
byte_string["emoticon"] = "happy"
try:
self.validate(byte_string)
except InputValidationError as err:
assert err.error["failed_test"] == "byte_data"
# test max_length
max_string = deepcopy(valid_input)
max_string["userID"] = "LongAlphaNumericID"
try:
self.validate(max_string)
except InputValidationError as err:
assert err.error["failed_test"] == "max_length"
# test min_length exception
min_string = deepcopy(valid_input)
min_string["userID"] = "ShortID"
try:
self.validate(min_string)
except InputValidationError as err:
assert err.error["failed_test"] == "min_length"
# test must_not_contain exception
prohibited_string = deepcopy(valid_input)
prohibited_string["userID"] = "6nPb/9gTwLz3f"
try:
self.validate(prohibited_string)
except InputValidationError as err:
assert err.error["failed_test"] == "must_not_contain"
# test must_contain exception
required_words = deepcopy(valid_input)
required_words["comments"][0] = "a"
try:
self.validate(required_words)
except InputValidationError as err:
assert err.error["failed_test"] == "must_contain"
# test contains_either exception
optional_words = deepcopy(valid_input)
optional_words["address"]["region"] = "N1"
try:
self.validate(optional_words)
except InputValidationError as err:
assert err.error["failed_test"] == "contains_either"
# test empty list
empty_list = deepcopy(valid_input)
empty_list["comments"] = []
input_value = self.keyMap[".comments"]["min_size"]
self.keyMap[".comments"]["min_size"] = 0
assert self.validate(empty_list)
self.keyMap[".comments"]["min_size"] = input_value
# test list reconstruction
assert isinstance(self._reconstruct(".comments"), list)
# test dict reconstruction
assert isinstance(self._reconstruct(".address"), dict)
# test nested reconstruction
assert isinstance(self._reconstruct(".address.country_code"), int)
assert isinstance(self._reconstruct(".comments[0]"), str)
# test ingest valid input
ingest_input = deepcopy(valid_input)
self.ingest(**ingest_input)
# test malformed dictionary datatype input ingestion
malformed_datatype = deepcopy(valid_input)
malformed_datatype["address"] = "my home"
valid_output = self.ingest(**malformed_datatype)
assert not valid_output["address"]["region"]
# test missing default input injection
missing_default = deepcopy(valid_input)
assert not "rating" in missing_default.keys()
valid_output = self.ingest(**missing_default)
assert valid_output["rating"] == 5
# test malformed default input replacement
malformed_default = deepcopy(valid_input)
malformed_default["rating"] = "5"
valid_output = self.ingest(**malformed_default)
assert valid_output["rating"] == 5
# test invalid default input replacement
invalid_default = deepcopy(valid_input)
invalid_default["rating"] = 11
valid_output = self.ingest(**invalid_default)
assert valid_output["rating"] == 5
# test missing input null injection
missing_string = deepcopy(valid_input)
del missing_string["userID"]
valid_output = self.ingest(**missing_string)
assert isinstance(valid_output["userID"], str)
assert not valid_output["userID"]
# test malformed input null injection
malformed_string = deepcopy(valid_input)
malformed_string["userID"] = {"key": "value"}
valid_output = self.ingest(**malformed_string)
assert isinstance(valid_output["userID"], str)
assert not valid_output["userID"]
# test invalid input null injection
invalid_string = deepcopy(valid_input)
invalid_string["userID"] = "tooShort"
valid_output = self.ingest(**invalid_string)
assert isinstance(valid_output["userID"], str)
assert not valid_output["userID"]
# test strip extra field input
ingest_input = deepcopy(extra_key_input)
valid_output = self.ingest(**ingest_input)
assert "extraKey" in ingest_input.keys()
assert not "extraKey" in valid_output.keys()
# test tag along of extra fields in input
self.keyMap["."]["extra_fields"] = True
ingest_input = deepcopy(extra_key_input)
valid_output = self.ingest(**ingest_input)
assert "extraKey" in ingest_input.keys()
assert "extraKey" in valid_output.keys()
self.keyMap["."]["extra_fields"] = False
# test mass injection of defaults and nulls
valid_output = self.ingest(**{})
for key in self.schema.keys():
assert key in valid_output.keys()
assert valid_output["rating"] == 5
assert not valid_output["userID"]
assert not valid_output["comments"]
assert valid_output["address"]
ex_int = 0
assert valid_output["address"]["country_code"].__class__ == ex_int.__class__
# test nested default injection
ingest_input = deepcopy(valid_input)
assert not "city" in ingest_input["address"].keys()
valid_output = self.ingest(**ingest_input)
assert valid_output["address"]["city"] == "New York"
# test max list length ingestion
long_list = deepcopy(valid_input)
long_list["comments"].insert(0, "pewter")
assert len(long_list["comments"]) == 4
valid_output = self.ingest(**long_list)
assert len(valid_output["comments"]) == 3
assert "bronze" not in valid_output["comments"]
# print(self.validate(valid_input))
print(self.ingest(**valid_input))
# print(self.ingest(**{}))
test_model = {
"schema": self.schema,
"components": self.components,
"metadata": self.metadata,
"title": self.title,
"description": self.description,
"max_size": self.maxSize,
"url": self.url,
}
# test list ingestion of dictionaries
new_model = deepcopy(test_model)
new_model["schema"]["test_list"] = [{"test": "me"}]
list_model = jsonModel(new_model)
test_input = {"test_list": [{"test": "me"}, {"test": "you"}]}
valid_output = list_model.ingest(**test_input)
assert valid_output["test_list"]
# test json valid structure of model components
assert json.dumps(test_model)
# test . use in key names
dot_key_names = deepcopy(test_model)
dot_key_names["schema"]["."] = {".": ""}
dot_key_names["components"][".."] = {"required_field": False}
dot_model = jsonModel(dot_key_names)
dot_ingested = dot_model.ingest(**{".": {".": "test"}})
assert dot_ingested["."]["."] == "test"
# test null value in model
null_schema = deepcopy(test_model)
null_schema["test"] = None
assert jsonModel(null_schema)
# test empty schema exception
empty_schema = deepcopy(test_model)
empty_schema["schema"] = {}
try:
jsonModel(empty_schema)
except ModelValidationError as err:
assert err
# test invalid json data exception
object_model = deepcopy(test_model)
object_model["schema"]["not_json"] = list_model
try:
jsonModel(object_model)
except ModelValidationError as err:
assert str(err).find(".not_json") > 0
object_model["schema"]["not_json"] = [list_model]
try:
jsonModel(object_model)
except ModelValidationError as err:
assert str(err).find(".not_json[0]") > 0
# test wrong datatype qualifier keys in components exception
integer_criteria_error = deepcopy(test_model)
integer_criteria_error["components"][".rating"]["must_not_contain"] = ["\\w"]
try:
jsonModel(integer_criteria_error)
except ModelValidationError as err:
assert str(err).find(".rating") > 0
# test wrong datatype qualifier values in components exception
contains_either_error = deepcopy(test_model)
contains_either_error["components"][".address.region"]["contains_either"].append(2)
try:
jsonModel(contains_either_error)
except ModelValidationError as err:
assert str(err).find(".address.region") > 0
# test wrong datatype qualifier values in components exception
min_size_error = deepcopy(test_model)
min_size_error["components"][".comments"]["min_size"] = -2
try:
jsonModel(min_size_error)
except ModelValidationError as err:
assert str(err).find(".comments") > 0
# test wrong datatype qualifier values in components exception
max_value_error = deepcopy(test_model)
max_value_error["components"][".rating"]["max_value"] = "10"
try:
jsonModel(max_value_error)
except ModelValidationError as err:
assert str(err).find(".rating") > 0
# test wrong datatype qualifier values in components exception
field_description_error = deepcopy(test_model)
field_description_error["components"][".userID"]["field_description"] = []
try:
jsonModel(field_description_error)
except ModelValidationError as err:
assert str(err).find(".userID") > 0
# test conflicting byte data and range criteria exception
byte_range_error = deepcopy(test_model)
byte_range_error["components"][".emoticon"]["min_value"] = "Ng=="
try:
jsonModel(byte_range_error)
except ModelValidationError as err:
assert str(err).find(".emoticon") > 0
# test conflicting greater_than and contains_either exception
value_contains_error = deepcopy(test_model)
value_contains_error["components"][".address.region"]["greater_than"] = "1B"
try:
jsonModel(value_contains_error)
except ModelValidationError as err:
assert str(err).find(".address.region") > 0
# test conflicting declared and excluded values in schema exception
excluded_value_error = deepcopy(test_model)
excluded_value_error["components"][".address.city"]["excluded_values"] = ["New Orleans"]
try:
jsonModel(excluded_value_error)
except ModelValidationError as err:
assert str(err).find(".address.city") > 0
# test conflicting discrete and excluded values in schema exception
discrete_value_error = deepcopy(test_model)
discrete_value_error["components"][".address.city"]["excluded_values"] = ["Miami"]
try:
jsonModel(discrete_value_error)
except ModelValidationError as err:
assert str(err).find(".address.city") > 0
# test item designator pattern used in schema keys
item_designator_error = deepcopy(test_model)
item_designator_error["schema"]["[1]"] = ""
try:
jsonModel(item_designator_error)
except ModelValidationError as err:
assert str(err).find(".[1]") > 0
# test query rules input
assert jsonModel(test_model, self.queryRules)
# test query rules json file
query_rules = json.loads(open("../models/query-rules.json").read())
assert jsonModel(test_model, query_rules)
# test query rules extra field exception
query_rules_field = deepcopy(self.queryRules)
query_rules_field[".none_fields"] = {}
try:
jsonModel(test_model, query_rules_field)
except ModelValidationError as err:
assert str(err).find(".string_fields") > 0
# test query rules extra qualifier exception
query_rules_qualifier = deepcopy(self.queryRules)
query_rules_qualifier[".string_fields"]["field_title"] = "not a qualifier"
try:
jsonModel(test_model, query_rules_qualifier)
except ModelValidationError as err:
assert str(err).find(".string_fields") > 0
# test query rules qualifier value exception
query_rules_value = deepcopy(self.queryRules)
query_rules_value[".string_fields"]["min_value"] = 0.0
try:
jsonModel(test_model, query_rules_value)
except ModelValidationError as err:
assert str(err).find(".string_fields") > 0
# test internal walk method
metals_list = ["gold", "silver", "bronze"]
metal_list = [{"metal": "gold"}, {"metal": "silver"}, {"metal": "bronze"}]
test_input = {"metals": deepcopy(metals_list)}
results = self._walk(".metals[0]", test_input)
assert len(results) == 3
test_input = {"metals": deepcopy(metal_list)}
results = self._walk(".metals[0].metal", test_input)
assert len(results) == 3
for group in test_input["metals"]:
group["metal"] = deepcopy(metals_list)
results = self._walk(".metals[0].metal[0]", test_input)
assert len(results) == 9
for group in test_input["metals"]:
group["metal"] = deepcopy(metal_list)
results = self._walk(".metals[0].metal[0].metal", test_input)
assert len(results) == 9
for metals in test_input["metals"]:
for metal in metals["metal"]:
metal["metal"] = deepcopy(metals_list)
results = self._walk(".metals[0].metal[0].metal[0]", test_input)
assert len(results) == 27
for metals in test_input["metals"]:
for metal in metals["metal"]:
metal["metal"] = deepcopy(metal_list)
# print(test_input)
results = self._walk(".metals[0].metal[0].metal[0].metal", test_input)
assert len(results) == 27
# print(results)
# test evaluate valid input
truth_table = []
for key, value in valid_query.items():
truth_table.append(self._evaluate_field(valid_input, key, value))
assert True in truth_table
assert False in truth_table
# print(valid_input)
# print(valid_query)
# print(truth_table)
# test evaluate query field missing in input
test_query = deepcopy(valid_query)
test_input = deepcopy(valid_input)
eval_kwargs = {
"record_dict": test_input,
"field_name": ".address.country_code",
"field_criteria": test_query[".address.country_code"],
}
eval_outcome = self._evaluate_field(**eval_kwargs)
assert not eval_outcome
# test evaluate query field missing with value_exists: false
eval_kwargs["field_criteria"]["value_exists"] = False
eval_outcome = self._evaluate_field(**eval_kwargs)
assert eval_outcome
# test evaluate query field exists in input
eval_kwargs["field_name"] = ".datetime"
eval_kwargs["field_criteria"] = test_query[".datetime"]
eval_outcome = self._evaluate_field(**eval_kwargs)
assert eval_outcome
# test evaluate query field exists with value_exists: false
eval_kwargs["field_criteria"]["value_exists"] = False
eval_outcome = self._evaluate_field(**eval_kwargs)
assert not eval_outcome
del eval_kwargs["field_criteria"]["value_exists"]
# test evaluate maximum size query failure
eval_kwargs["field_name"] = ".comments"
eval_kwargs["field_criteria"] = test_query[".comments"]
eval_kwargs["field_criteria"]["max_size"] = 2
eval_outcome = self._evaluate_field(**eval_kwargs)
assert not eval_outcome
del eval_kwargs["field_criteria"]["max_size"]
# test evaluate unique values query failure
eval_kwargs["record_dict"]["comments"].append("gold")
eval_outcome = self._evaluate_field(**eval_kwargs)
assert not eval_outcome
eval_kwargs["record_dict"]["comments"].pop()
# test evaluate min length query failure
eval_kwargs["field_name"] = ".userID"
eval_kwargs["field_criteria"] = test_query[".userID"]
eval_kwargs["field_criteria"]["min_length"] = 14
del eval_kwargs["field_criteria"]["min_value"]
eval_kwargs["field_criteria"]["max_length"] = 14
del eval_kwargs["field_criteria"]["max_value"]
eval_outcome = self._evaluate_field(**eval_kwargs)
assert not eval_outcome
# test evaluate max value query failure
eval_kwargs["field_criteria"]["min_length"] = 12
eval_kwargs["field_criteria"]["max_length"] = 14
eval_kwargs["field_criteria"]["max_value"] = "2222222222222"
eval_outcome = self._evaluate_field(**eval_kwargs)
assert not eval_outcome
# test evaluate less than query failure
eval_kwargs["field_name"] = ".datetime"
eval_kwargs["field_criteria"] = test_query[".datetime"]
eval_kwargs["field_criteria"]["less_than"] = 200000.0
eval_outcome = self._evaluate_field(**eval_kwargs)
assert not eval_outcome
eval_kwargs["field_criteria"]["less_than"] = 2000000000.0
# test evaluate integer only query failure
eval_kwargs["field_criteria"]["integer_data"] = True
eval_outcome = self._evaluate_field(**eval_kwargs)
assert not eval_outcome
del eval_kwargs["field_criteria"]["integer_data"]
# test evaluate excluded values query failure
eval_kwargs["field_name"] = (".emoticon",)
eval_kwargs["field_criteria"] = test_query[".emoticon"]
eval_kwargs["field_criteria"]["excluded_values"].append("aGFwcHk=")
eval_outcome = self._evaluate_field(**eval_kwargs)
assert not eval_outcome
eval_kwargs["field_criteria"]["excluded_values"].pop()
# test evaluate discrete value query failure
eval_kwargs["field_name"] = (".address.region",)
eval_kwargs["field_criteria"] = test_query[".address.region"]
eval_kwargs["field_criteria"]["discrete_values"] = ["CA", "MA"]
eval_outcome = self._evaluate_field(**eval_kwargs)
assert not eval_outcome
del eval_kwargs["field_criteria"]["discrete_values"]
# test evaluate byte data query failure
eval_kwargs["field_criteria"]["byte_data"] = True
eval_outcome = self._evaluate_field(**eval_kwargs)
assert not eval_outcome
del eval_kwargs["field_criteria"]["byte_data"]
# test evaluate contains either query failure
eval_kwargs["field_criteria"]["contains_either"][0] = "[A-Z]{3}"
eval_outcome = self._evaluate_field(**eval_kwargs)
assert not eval_outcome
eval_kwargs["field_criteria"]["contains_either"][0] = "[A-Z]{2}"
# test evaluate must contain query failure
eval_kwargs["field_name"] = (".address.country",)
eval_kwargs["field_criteria"] = {"must_contain": ["America"]}
eval_outcome = self._evaluate_field(**eval_kwargs)
assert not eval_outcome
del eval_kwargs["field_criteria"]["must_contain"]
# test evaluate must not contain query failure
eval_kwargs["field_criteria"] = {"must_not_contain": ["States"]}
eval_outcome = self._evaluate_field(**eval_kwargs)
assert not eval_outcome
# test sample empty query
assert self.query(valid_query)
assert isinstance(self.query(valid_query), bool)
# test query criteria invalid qualifier exception
query_qualifier_error = deepcopy(valid_query)
query_qualifier_error[".address.region"]["required_field"] = False
try:
self.query(query_qualifier_error)
except QueryValidationError as err:
assert str(err).find(".address.region") > 0
# test query criteria value exists exception
query_value_exists = deepcopy(valid_query)
query_value_exists[".address.region"]["value_exists"] = True
self.query(query_value_exists)
query_value_exists[".address.region"]["value_exists"] = False
try:
self.query(query_value_exists)
except QueryValidationError as err:
assert str(err).find("value_exists:") > 0
# test query method with valid query on valid input
assert not self.query(valid_query, valid_input)
assert isinstance(self.query(valid_query, valid_input), bool)
# test query method with non-existent field
assert self.query({".rating": {"value_exists": False}}, valid_input)
assert not self.query({".rating": {"value_exists": True}}, valid_input)
# test query method with number field queries
assert self.query({".datetime": {"value_exists": True}}, valid_input)
assert not self.query({".datetime": {"value_exists": False}}, valid_input)
assert self.query({".datetime": {"min_value": 1.1}}, valid_input)
assert not self.query({".datetime": {"min_value": 1500000000}}, valid_input)
assert self.query({".datetime": {"max_value": 1500000000}}, valid_input)
assert not self.query({".datetime": {"max_value": 1.1}}, valid_input)
assert self.query({".datetime": {"integer_data": False}}, valid_input)
assert not self.query({".datetime": {"integer_data": True}}, valid_input)
test_input = deepcopy(valid_input)
test_input["datetime"] = 50
assert not self.query({".datetime": {"integer_data": False}}, test_input)
assert self.query({".datetime": {"integer_data": True}}, test_input)
assert not self.query({".datetime": {"greater_than": 1449179763.312077}}, valid_input)
assert not self.query({".datetime": {"less_than": 1449179763.312077}}, valid_input)
assert self.query({".datetime": {"discrete_values": [1449179763.312077]}}, valid_input)
assert not self.query({".datetime": {"excluded_values": [1449179763.312077]}}, valid_input)
# test query method with string field queries
assert self.query({".userID": {"value_exists": True}}, valid_input)
assert not self.query({".userID": {"value_exists": False}}, valid_input)
assert self.query({".userID": {"min_length": 2}}, valid_input)
assert not self.query({".userID": {"min_length": 14}}, valid_input)
assert not self.query({".userID": {"max_length": 2}}, valid_input)
assert self.query({".userID": {"max_length": 14}}, valid_input)
assert self.query({".userID": {"min_value": "11111111111"}}, valid_input)
assert not self.query({".userID": {"min_value": "zzzzzzzzzzz"}}, valid_input)
assert not self.query({".userID": {"max_value": "11111111111"}}, valid_input)
assert self.query({".userID": {"max_value": "zzzzzzzzzzz"}}, valid_input)
assert not self.query({".userID": {"greater_than": "6nPbM9gTwLz3f"}}, valid_input)
assert self.query({".userID": {"greater_than": "6nPbM9gTwLz"}}, valid_input)
assert not self.query({".userID": {"less_than": "6nPbM9gTwLz3f"}}, valid_input)
assert self.query({".userID": {"less_than": "6nPbM9gTwLz3g"}}, valid_input)
assert self.query({".userID": {"discrete_values": ["6nPbM9gTwLz3f"]}}, valid_input)
assert not self.query({".userID": {"discrete_values": ["6nPbM9gTwLz"]}}, valid_input)
assert not self.query({".userID": {"excluded_values": ["6nPbM9gTwLz3f"]}}, valid_input)
assert self.query({".userID": {"excluded_values": ["6nPbM9gTwLz"]}}, valid_input)
assert self.query({".userID": {"must_contain": ["6nPbM9gTwLz"]}}, valid_input)
assert not self.query({".userID": {"must_contain": ["/"]}}, valid_input)
assert not self.query({".userID": {"must_not_contain": ["6nPbM9gTwLz"]}}, valid_input)
assert self.query({".userID": {"must_not_contain": ["/"]}}, valid_input)
assert self.query({".userID": {"contains_either": ["6nPbM9gTwLz", "/"]}}, valid_input)
assert not self.query({".userID": {"contains_either": [":", "/"]}}, valid_input)
assert self.query({".emoticon": {"byte_data": True}}, valid_input)
assert not self.query({".userID": {"byte_data": True}}, valid_input)
assert self.query({".userID": {"byte_data": False}}, valid_input)
assert not self.query({".emoticon": {"byte_data": False}}, valid_input)
# test query method with list field queries
assert self.query({".comments": {"value_exists": True}}, valid_input)
assert not self.query({".comments": {"value_exists": False}}, valid_input)
assert self.query({".comments": {"min_size": 2}}, valid_input)
assert not self.query({".comments": {"min_size": 4}}, valid_input)
assert not self.query({".comments": {"max_size": 2}}, valid_input)
assert self.query({".comments": {"max_size": 4}}, valid_input)
assert self.query({".comments": {"unique_values": True}}, valid_input)
assert not self.query({".comments": {"unique_values": False}}, valid_input)
test_input = deepcopy(valid_input)
test_input["comments"].append("gold")
assert not self.query({".comments": {"unique_values": True}}, test_input)
assert self.query({".comments": {"unique_values": False}}, test_input)
# test items in list field queries
assert self.query({".comments[0]": {"value_exists": True}}, valid_input)
assert not self.query({".comments[0]": {"value_exists": False}}, valid_input)
assert self.query({".comments[0]": {"min_length": 3}}, valid_input)
assert not self.query({".comments[0]": {"min_length": 5}}, valid_input)
assert self.query({".comments[0]": {"must_contain": [".{2}"]}}, valid_input)
assert not self.query({".comments[0]": {"must_contain": ["g.{2}"]}}, valid_input)
assert self.query({".comments[0]": {"contains_either": ["l", "o"]}}, valid_input)
assert not self.query({".comments[0]": {"contains_either": ["r", "e"]}}, valid_input)
return self
def __init__(self, instance_id, pem_file, access_id, secret_key, region_name, owner_id, user_name, login_name='', verbose=True):
'''
a method for initializing the SSH connection parameters to the EC2 instance
:param instance_id: string with AWS id of instance
:param pem_file: string with path to keypair pem file
:param access_id: string with access_key_id from aws IAM user setup
:param secret_key: string with secret_access_key from aws IAM user setup
:param region_name: string with name of aws region
:param owner_id: string with aws account id
:param user_name: string with name of user access keys are assigned to
:param login_name: [optional] string with name of login user
:param verbose: boolean to enable process messages
'''
title = '%s.__init__' % self.__class__.__name__
# initialize model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# construct localhost client
from labpack.platforms.localhost import localhostClient
self.localhost = localhostClient()
# validate credentials and construct ec2 method
from labpack.platforms.aws.ec2 import ec2Client
self.ec2 = ec2Client(access_id, secret_key, region_name, owner_id, user_name, verbose)
# validate inputs
input_fields = {
'instance_id': instance_id,
'pem_file': pem_file
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.ec2.fields.validate(value, '.%s' % key, object_title)
# construct class properties
self.instance_id = instance_id
# verify pem file exists
from os import path
if not path.exists(pem_file):
raise Exception('%s is not a valid path.' % pem_file)
self.pem_file = path.abspath(pem_file)
# verify user has privileges
try:
self.ec2.iam.printer_on = False
self.ec2.list_keypairs()
except AWSConnectionError as err:
if str(err).find('Could not connect') > -1:
raise
raise AWSConnectionError(title, 'You must have privileges to access EC2 to use sshClient')
# verify instance exists
instance_list = self.ec2.list_instances()
if instance_id not in instance_list:
raise Exception('%s does not exist in this region or permission scope.' % instance_id)
# verify instance has public ip
instance_details = self.ec2.read_instance(instance_id)
if not instance_details['public_ip_address']:
raise Exception('%s requires a public IP address to access through ssh.' % instance_id)
self.instance_ip = instance_details['public_ip_address']
# retrieve login name from tag
self.login_name = ''
input_fields = {
'login_name': login_name
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
self.login_name = login_name
if not self.login_name:
for tag in instance_details['tags']:
if tag['key'] == 'UserName':
self.login_name = tag['value']
if not self.login_name:
raise Exception('SSH access to %s requires a login_name argument or UserName tag' % instance_id)
# verify local and remote pem file names match
from os import path
pem_absolute = path.abspath(pem_file)
pem_root, pem_ext = path.splitext(pem_absolute)
pem_path, pem_name = path.split(pem_root)
if not instance_details['key_name'] == pem_name:
raise Exception('%s does not match name of keypair %s for instance %s.' % (pem_name, instance_details['keypair'], instance_id))
# verify instance is ready
self.ec2.check_instance_status(instance_id)
# verify security group allows ssh
group_list = []
for group in instance_details['security_groups']:
group_list.append(group['group_id'])
if not group_list:
raise Exception('SSH access to %s requires a security group attached to instance.' % instance_id)
port_22_list = []
for group_id in group_list:
group_details = self.ec2.read_security_group(group_id)
for permission in group_details['ip_permissions']:
if permission['from_port'] == 22:
port_22_list.extend(permission['ip_ranges'])
if not port_22_list:
raise Exception('SSH access to %s requires a security group with inbound permissions for port 22.' % instance_id)
from labpack.records.ip import get_ip
current_ip = get_ip()
ssh_access = False
for ip_range in port_22_list:
if ip_range['cidr_ip'].find('0.0.0.0/0') > -1 or ip_range['cidr_ip'].find(current_ip) > -1:
ssh_access = True
break
if not ssh_access:
raise Exception('SSH access to %s requires your IP %s to be added to port 22 on its security group.' % (instance_id, current_ip))
# verify pem file has access
try:
self.script('ls -a')
except:
raise AWSConnectionError(title, '%s does not have access to instance %s.' % (pem_name, instance_id))
# verify scp
self.scp = False
# turn printer back on
self.ec2.iam.printer_on = True
def positional_filter(positional_filters, title=''):
'''
a method to construct a conditional filter function to test positional arguments
:param positional_filters: dictionary or list of dictionaries with query criteria
:param title: string with name of function to use instead
:return: callable for filter_function
NOTE: query criteria architecture
each item in the path filters argument must be a dictionary
which is composed of integer-value key names that represent the
index value of the positional segment to test and key values
with the dictionary of conditional operators used to test the
string value in the indexed field of the record.
eg. positional_filters = [ { 0: { 'must_contain': [ '^lab' ] } } ]
this example filter looks at the first segment of each key string
in the collection for a string value which starts with the
characters 'lab'. as a result, it will match both the following:
lab/unittests/1473719695.2165067.json
'laboratory20160912.json'
NOTE: the filter function uses a query filters list structure to represent
the disjunctive normal form of a logical expression. a record is
added to the results list if any query criteria dictionary in the
list evaluates to true. within each query criteria dictionary, all
declared conditional operators must evaluate to true.
in this way, the positional_filters represents a boolean OR operator and
each criteria dictionary inside the list represents a boolean AND
operator between all keys in the dictionary.
each query criteria uses the architecture of query declaration in
the jsonModel.query method
NOTE: function function will lazy load a dictionary input
positional_filters:
[ { 0: { conditional operators }, 1: { conditional_operators }, ... } ]
conditional operators:
"byte_data": false,
"discrete_values": [ "" ],
"excluded_values": [ "" ],
'equal_to': '',
"greater_than": "",
"less_than": "",
"max_length": 0,
"max_value": "",
"min_length": 0,
"min_value": "",
"must_contain": [ "" ],
"must_not_contain": [ "" ],
"contains_either": [ "" ]
'''
# define help text
if not title:
title = 'positional_filter'
filter_arg = '%s(positional_filters=[...])' % title
# construct path_filter model
filter_schema = {
'schema': {
'byte_data': False,
'discrete_values': [ '' ],
'excluded_values': [ '' ],
'equal_to': '',
'greater_than': '',
'less_than': '',
'max_length': 0,
'max_value': '',
'min_length': 0,
'min_value': '',
'must_contain': [ '' ],
'must_not_contain': [ '' ],
'contains_either': [ '' ]
},
'components': {
'.discrete_values': {
'required_field': False
},
'.excluded_values': {
'required_field': False
},
'.must_contain': {
'required_field': False
},
'.must_not_contain': {
'required_field': False
},
'.contains_either': {
'required_field': False
}
}
}
from jsonmodel.validators import jsonModel
filter_model = jsonModel(filter_schema)
# lazy load path dictionary
if isinstance(positional_filters, dict):
positional_filters = [ positional_filters ]
# validate input
if not isinstance(positional_filters, list):
raise TypeError('%s must be a list.' % filter_arg)
for i in range(len(positional_filters)):
if not isinstance(positional_filters[i], dict):
raise TypeError('%s item %s must be a dictionary.' % (filter_arg, i))
for key, value in positional_filters[i].items():
_key_name = '%s : {...}' % key
if not isinstance(key, int):
raise TypeError('%s key name must be an int.' % filter_arg.replace('...', _key_name))
elif not isinstance(value, dict):
raise TypeError('%s key value must be a dictionary' % filter_arg.replace('...', _key_name))
filter_model.validate(value)
# construct segment value model
segment_schema = { 'schema': { 'segment_value': 'string' } }
segment_model = jsonModel(segment_schema)
# construct filter function
def filter_function(*args):
max_index = len(args) - 1
for filter in positional_filters:
criteria_match = True
for key, value in filter.items():
if key > max_index:
criteria_match = False
break
segment_criteria = { '.segment_value': value }
segment_data = { 'segment_value': args[key] }
if not segment_model.query(segment_criteria, segment_data):
criteria_match = False
break
if criteria_match:
return True
return False
return filter_function
def __init__(self, hostname, port=9042, username='', password='', cert_path=''):
title = '%s.__init__' % self.__class__.__name__
# construct fields model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# ingest hostname
self_hostname = hostname
if isinstance(hostname, str):
if hostname:
self_hostname = [ hostname ]
# validate inputs
input_fields = {
'hostname': self_hostname,
'port': port,
'username': username,
'password': password,
'cert_path': cert_path
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct endpoint
self.hostname = self_hostname
self.port = port
self.username = username
self.password = password
self.cert_path = cert_path
# construct cluster
cluster_kwargs = {
'contact_points': self.hostname,
'port': self.port
}
if self.username and self.password:
from sys import path as sys_path
sys_path.append(sys_path.pop(0))
from cassandra.auth import PlainTextAuthProvider
sys_path.insert(0, sys_path.pop())
cluster_kwargs['auth_provider'] = PlainTextAuthProvider(
username=username,
password=password
)
if cert_path:
from os import path
import ssl
if not path.exists(cert_path):
raise ValueError('%s(cert_path="%s") is not a valid file path.' % (title, cert_path))
cluster_kwargs['ssl_options'] = {
'ca_certs': cert_path,
'cert_reqs': ssl.CERT_REQUIRED,
'ssl_version': ssl.PROTOCOL_TLSv1
}
self.cluster = Cluster(**cluster_kwargs)
# construct session
self.session = self.cluster.connect()
def __init__(self, magic_file=''):
''' initialization method for labMagic class
:param magic_file: [optional] string with local path to magic.mgc file
'''
title = '%s.__init__' % self.__class__.__name__
# construct class field model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'magic_file': magic_file
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct magic method
magic_kwargs = {
'mime': True,
'uncompress': True
}
from labpack.platforms.localhost import localhostClient
sys_name = localhostClient().os.sysname
if sys_name == 'Windows':
if not magic_file:
raise IndexError('%s(magic_file="...") is required on Windows systems.')
import os
if magic_file:
if not os.path.exists(magic_file):
raise ValueError('%s(magic_file=%s) is not a valid file path.' % (title, magic_file))
magic_kwargs['magic_file'] = magic_file
try:
# workaround for module namespace conflict
from sys import path as sys_path
sys_path.append(sys_path.pop(0))
import magic
sys_path.insert(0, sys_path.pop())
self.magic = magic.Magic(**magic_kwargs)
except:
raise Exception('\nmagiclab requires the python-magic module. try: pip install python-magic\npython-magic requires the C library libmagic. See documentation in labpack.parsing.magic.')
# construct mimetypes method
import mimetypes
self.mimetypes = mimetypes.MimeTypes()
# retrieve updates to mimetypes
mimetype_urls = self.fields.schema['mimetype_urls']
from labpack.storage.appdata import appdataClient
mime_collection = appdataClient('Mime Types')
mime_filter = mime_collection.conditional_filter([{-1:{'must_contain': ['mime.types']}}])
mime_list = mime_collection.list(mime_filter)
for key in mimetype_urls.keys():
file_path = os.path.join(mime_collection.collection_folder, key)
if key not in mime_list:
file_dir = os.path.split(file_path)[0]
if not os.path.exists(file_dir):
os.makedirs(file_dir)
import requests
try:
response = requests.get(mimetype_urls[key])
except Exception:
from labpack.handlers.requests import handle_requests
request_kwargs = {'url': mimetype_urls[key]}
response_details = handle_requests(requests.Request(**request_kwargs))
print('magiclab attempted to retrieve latest mimetype registry resource at %s but ran into this non-fatal error: %s' % (mimetype_urls[key], response_details['error']))
break
with open(file_path, 'wb') as f:
f.write(response.content)
f.close()
ext_map = mimetypes.read_mime_types(file_path)
for key, value in ext_map.items():
self.mimetypes.add_type(value, key)
def _apply_update(root_path, service_name=''):
# construct message
msg_insert = 'local service'
if service_name:
msg_insert = 'service "%s"' % service_name
# update vcs ignore
import hashlib
from os import path
from pocketlab.methods.vcs import merge_ignores
vcs_files = {
'git': {
'path': path.join(root_path, '.gitignore'),
'name': '.gitignore'
},
'mercurial': {
'path': path.join(root_path, '.hgignore'),
'name': '.hgignore'
}
}
for key, value in vcs_files.items():
if path.exists(value['path']):
old_text = open(value['path']).read()
old_hash = hashlib.sha1(old_text.encode('utf-8')).hexdigest()
new_text = merge_ignores(old_text, vcs_templates[key])
new_hash = hashlib.sha1(new_text.encode('utf-8')).hexdigest()
if old_hash != new_hash:
with open(value['path'], 'wt') as f:
f.write(new_text)
f.close()
if verbose:
print('%s file for %s updated.' % (value['name'], msg_insert))
# update lab yaml
from pocketlab import __module__
from jsonmodel.loader import jsonLoader
from jsonmodel.validators import jsonModel
from labpack.records.settings import save_settings, load_settings
config_schema = jsonLoader(__module__, 'models/lab-config.json')
config_model = jsonModel(config_schema)
template_config = config_model.ingest()
config_path = path.join(root_path, 'lab.yaml')
if path.exists(config_path):
try:
old_config = load_settings(config_path)
template_config.update(**old_config)
if old_config != template_config:
from pocketlab.methods.config import compile_yaml
config_text = compile_yaml(config_schema, config_path)
with open(config_path, 'wt') as f:
f.write(config_text)
f.close()
if verbose:
print('lab.yaml file for %s updated.' % msg_insert)
except:
print('lab.yaml file for %s is corrupted. Skipped.' % msg_insert)
# update setup.py
setup_path = path.join(root_path, 'setup.py')
if path.exists(setup_path):
from pocketlab.methods.config import update_setup
old_text = open(setup_path).read()
old_hash = hashlib.sha1(old_text.encode('utf-8')).hexdigest()
new_text = update_setup(old_text)
new_hash = hashlib.sha1(new_text.encode('utf-8')).hexdigest()
if old_hash != new_hash:
with open(setup_path, 'wt', encoding='utf-8') as f:
f.write(new_text)
f.close()
if verbose:
print('setup.py file for %s updated.' % msg_insert)