def __init__(self, access_token, collection_name=''): ''' a method to initialize the driveClient class :param access_token: string with oauth2 access token for users account :param collection_name: [optional] string with name of collection for import ''' title = '%s.__init__' % self.__class__.__name__ # construct input validation model self.fields = jsonModel(self._class_fields) # validate inputs input_fields = { 'access_token': access_token, 'collection_name': collection_name } for key, value in input_fields.items(): object_title = '%s(%s=%s)' % (title, key, str(value)) self.fields.validate(value, '.%s' % key, object_title) # construct access token self.access_token = access_token # construct drive client import httplib2 from googleapiclient import discovery from oauth2client.client import AccessTokenCredentials google_credentials = AccessTokenCredentials(self.access_token, 'my-user-agent/1.0') google_http = httplib2.Http() google_http = google_credentials.authorize(google_http) google_drive = discovery.build('drive', 'v3', http=google_http) self.drive = google_drive.files() # construct collection properties self.permissions_write = True self.permissions_content = True self.drive_space = 'drive' self.space_id = '' if collection_name: self.collection_name = collection_name else: self.collection_name = 'My Drive' # construct file object from labpack import __module__ from jsonmodel.loader import jsonLoader drive_rules = jsonLoader(__module__, 'storage/google/drive-rules.json') self.object_file = drive_rules['file_object'] # validate access token self._validate_token()
def __init__(self, access_id, secret_key, region_name, owner_id, user_name, verbose=True, usage_client=None): ''' a method for initializing the connection to AWS Polly :param access_id: string with access_key_id from aws IAM user setup :param secret_key: string with secret_access_key from aws IAM user setup :param region_name: string with name of aws region :param owner_id: string with aws account id :param user_name: string with name of user access keys are assigned to :param verbose: boolean to enable process messages :param usage_client: callable object to track resource usage ''' title = '%s.__init__' % self.__class__.__name__ # initialize model from labpack import __module__ from jsonmodel.loader import jsonLoader from jsonmodel.validators import jsonModel class_fields = jsonLoader(__module__, 'speech/aws/polly-rules.json') self.fields = jsonModel(class_fields) # construct iam connection from labpack.authentication.aws.iam import iamClient self.iam = iamClient(access_id, secret_key, region_name, owner_id, user_name, verbose) # construct polly client connection client_kwargs = { 'service_name': 'polly', 'region_name': self.iam.region_name, 'aws_access_key_id': self.iam.access_id, 'aws_secret_access_key': self.iam.secret_key } self.connection = boto3.client(**client_kwargs) self.verbose = verbose self.usage_client = usage_client # construct range of polly options self.voice_ids = self.fields.components['.voice_id']['discrete_values'] self.output_formats = self.fields.components['.output_format']['discrete_values'] # construct pythonic conversion method from labpack.parsing.conversion import camelcase_to_lowercase self.ingest = camelcase_to_lowercase
def start(service_list, verbose=True, virtualbox='default'): title = 'start' # validate inputs if isinstance(service_list, str): if service_list: service_list = [service_list] input_fields = { 'service_list': service_list, 'verbose': verbose, 'virtualbox': virtualbox } for key, value in input_fields.items(): if value: object_title = '%s(%s=%s)' % (title, key, str(value)) fields_model.validate(value, '.%s' % key, object_title) # validate installation of docker from pocketlab.methods.docker import dockerClient docker_client = dockerClient(virtualbox_name=virtualbox, verbose=verbose) # verbosity if verbose: print('Checking service configurations...', end='', flush=True) # construct list of paths to services from pocketlab.methods.service import retrieve_services start_list, msg_insert = retrieve_services(service_list) # construct lab list lab_list = [] # validate lab files from os import path from pocketlab.methods.validation import validate_lab from pocketlab import __module__ from jsonmodel.loader import jsonLoader from jsonmodel.validators import jsonModel lab_model = jsonModel(jsonLoader(__module__, 'models/lab-config.json')) for details in start_list: file_path = path.join(details['path'], 'lab.yaml') service_name = details['name'] lab_details = validate_lab(lab_model, file_path, service_name) details['config'] = lab_details lab_list.append(details) # retrieve list of docker images # validate lab config image exists # retrieve list of docker containers # validate lab config container doesn't already exist # TODO resolve name conflicts on deploy updating # validate mount paths exist if verbose: print('.', end='', flush=True) # end validations if verbose: print(' done.') # retrieve system ip system_ip = docker_client.ip() # instantiate containers exit_msg = '' port_list = [] for service in lab_list: service_config = service['config'] service_name = service['name'] service_root = service['path'] # construct environment variables container_envvar = { 'SYSTEM_IP_ADDRESS': system_ip } if service_config['docker_environment_variables']: for key, value in service_config['docker_environment_variables'].items(): container_envvar[key.upper()] = '"%s"' % value # construct sys command container_name = service_config['docker_container_alias'] sys_command = 'docker run --name %s' % container_name for key, value in container_envvar.items(): sys_command += ' -e %s=%s' % (key, value) if service_config['docker_mount_volumes']: for key, value in service_config['docker_mount_volumes'].items(): sys_command += ' -v "%s":"%s"' % (key, value) sys_command += ' -it -d' port_count = 0 port_value = '' if service_config['docker_port_mapping']: for key, value in service_config['docker_port_mapping'].items(): port_value = value if value in port_list: from copy import deepcopy port_value = deepcopy(value) for i in range(1000): port_value += 1 if not port_value in port_list: break port_list.append(port_value) port_count += 1 sys_command += ' -p %s:%s' % (str(port_value), key) sys_command += ' %s' % service_config['docker_image_name'] # determine port message port_msg = '' if port_count == 1: port_msg = ' at %s:%s' % (system_ip, port_value) # run command from subprocess import check_output docker_response = check_output(sys_command).decode('utf-8') service_msg = 'Container "%s" started%s' % (container_name, port_msg) if len(lab_list) > 1: if verbose: print(service_msg) else: exit_msg = service_msg # TODO consider ROLLBACK options if len(lab_list) > 1: exit_msg = 'Finished starting %s' % msg_insert return exit_msg # # import dependencies # from os import path # from copy import deepcopy # from pocketlab.importers.config_file import configFile # from pocketlab.clients.localhost_client import localhostClient # from pocketlab.clients.docker_session import dockerSession # from pocketlab.validators.config_model import configModel # from pocketlab.validators.absolute_path import absolutePath # from pocketlab.validators.available_image import availableImage # from pocketlab.compilers.docker_run import dockerRun # # # ingest verbose options # verbose = kwargs['verbose'] # # # determine system properties # localhost = localhostClient() # # # ingest & validate component file # component_file = kwargs['componentFile'] # comp_details = configFile(component_file, kwargs) # comp_details = configModel(comp_details, 'rules/lab-component-model.json', kwargs, 'component settings') # # # determine component root from component file # root_path, file_name = path.split(path.abspath(component_file)) # # # construct path details to mounted volumes # mounted_volumes = {} # for volume in comp_details['mounted_volumes']: # host_path = absolutePath(volume, root_path, kwargs, 'mounted volume') # system_path = host_path.replace('\\','/').replace('C:','//c') # absolute_path = deepcopy(host_path) # docker_path = absolute_path.replace(root_path,'') # container_path = docker_path.replace('\\','/') # mounted_volumes[system_path] = container_path # # # ingest & validate virtualbox property # vbox_name = kwargs['virtualbox'] # if not localhost.os in ('Windows','Mac'): # vbox_name = '' # # # check for docker installation # docker_session = dockerSession(kwargs, vbox_name) # # # check that docker image is available locally # image_list = docker_session.images() # availableImage(comp_details['docker_image'], comp_details['image_tag'], image_list, kwargs) # # # retrieve list of active container aliases & busy ports # container_list = docker_session.ps() # busy_ports = [] # active_containers = [] # for container in container_list: # active_containers.append(container['NAMES']) # container_settings = docker_session.inspect(container_alias=container['NAMES']) # container_synopsis = docker_session.synopsis(container_settings) # if container_synopsis['mapped_ports']: # for key in container_synopsis['mapped_ports'].keys(): # busy_ports.append(key) # # # check that alias name is available # if comp_details['container_alias'] in active_containers: # from pocketlab.exceptions.lab_exception import labException # header_list = [ 'NAMES', 'STATUS', 'IMAGE', 'PORTS'] # error = { # 'kwargs': kwargs, # 'message': 'Container "%s" already in use. Containers currently active:' % comp_details['container_alias'], # 'tprint': { 'headers': header_list, 'rows': container_list }, # 'error_value': comp_details['container_alias'], # 'failed_test': 'unavailable_resource' # } # raise labException(**error) # # # construct port mappings for mapped ports # mapped_ports = {} # for port in comp_details['exposed_ports']: # open_port = int(deepcopy(port)) # while str(open_port) in busy_ports: # open_port += 1 # mapped_ports[str(open_port)] = str(port) # # # add system_ip to injected variables # system_ip = docker_session.ip() # injected_variables = { # 'SYSTEM_LOCALHOST': system_ip # } # for key, value in comp_details['injected_variables'].items(): # injected_variables[key] = value # # # compile docker run script from settings # run_details = { # 'name': comp_details['container_alias'], # 'injected_variables': injected_variables, # 'mounted_volumes': mounted_volumes, # 'mapped_ports': mapped_ports, # 'docker_image': comp_details['docker_image'], # 'image_tag': comp_details['image_tag'], # 'run_command': comp_details['run_command'] # } # run_script = dockerRun(run_details) # # # start container # container_id = docker_session.run(run_script) # if verbose: # start_text = 'Sweet! Container "%s" started' % comp_details['container_alias'] # if run_details['mapped_ports']: # start_text += ' on port' # if len(run_details['mapped_ports'].keys()) > 1: # start_text += 's' # previous_port = False # for key in run_details['mapped_ports'].keys(): # if previous_port: # start_text += ',' # start_text += ' %s:%s' % (system_ip, key) # previous_port = True # start_text += '.' # print(start_text) # # container_details = { # 'mapped_ports': mapped_ports, # 'container_alias': comp_details['container_alias'], # 'container_id': container_id # } # # return container_details
def __init__(self, bucket_name, database_url, document_schema=None, verbose=False, public=False, configs=None): ''' the initialization method for syncGatewayAdmin class ''' # https://developer.couchbase.com/documentation/mobile/1.5/guides/sync-gateway/config-properties/index.html # https://developer.couchbase.com/documentation/mobile/current/guides/sync-gateway/sync-function-api-guide/index.html title = '%s.__init__' % self.__class__.__name__ # import default sync function from os import path from importlib.util import find_spec module_path = find_spec(__module__).submodule_search_locations[0] sync_path = path.join(module_path, 'databases/models/sync_function.js') sync_text = open(sync_path).read() sync_text = self._update_js(sync_text, document_schema) # construct fields self._class_fields['components']['.configs.sync']['default_value'] = sync_text from jsonmodel.validators import jsonModel self.fields = jsonModel(self._class_fields) # validate inputs input_fields = { 'bucket_name': bucket_name, 'database_url': database_url, 'document_schema': document_schema, 'configs': configs } for key, value in input_fields.items(): if value: object_title = '%s(%s=%s)' % (title, key, str(value)) self.fields.validate(value, '.%s' % key, object_title) # test connection to db self.admin_access = True try: response = requests.get(database_url) response = response.json() if not 'ADMIN' in response.keys(): self.admin_access = False except: raise Exception('%s(database_url="%s") is not a valid couchbase url.' % (title, database_url)) # construct class properties from os import path self.bucket_name = bucket_name self.database_url = database_url self.bucket_url = path.join(database_url, bucket_name) # construct verbose method self.printer_on = True def _printer(msg, flush=False): if verbose and self.printer_on: if flush: print(msg, end='', flush=True) else: print(msg) self.printer = _printer # construct document model from jsonmodel.validators import jsonModel self.public = public self.model = None if document_schema: self.model = jsonModel(document_schema) elif not self.public: self.model = jsonModel({'schema': { 'uid': 'abc012XYZ789' }, 'components': {'.': { 'extra_fields': True}}}) if self.model: if not 'uid' in self.model.schema.keys(): document_schema['schema']['uid'] = 'abc012XYZ789' if not self.model.components: document_schema['components'] = { '.': { 'extra_fields': True } } elif not '.' in self.model.components.keys(): document_schema['components']['.'] = { 'extra_fields': True } elif not 'extra_fields' in self.model.components['.'].keys(): document_schema['components']['.']['extra_fields'] = True self.model = jsonModel(document_schema) # construct configs default_fields = self.fields.ingest(**{}) self.configs = default_fields['configs'] self.configs['bucket'] = self.bucket_name self.configs['name'] = self.bucket_name if self.public: from jsonmodel.loader import jsonLoader read_only_configs = jsonLoader(__module__, 'databases/models/sync-gateway-public.json') for key, value in read_only_configs.items(): self.configs[key] = value if configs: for key, value in configs.items(): self.configs[key] = value # construct lab record generator from labpack.records.id import labID self.labID = labID # create db (update config) if none exists self._update_bucket()
def unitTests(self): # test invalid module exception with pytest.raises(Exception): jsonLoader('not_a_module', 'model-rules.json') # test valid relative paths assert jsonLoader(__module__, 'model-rules.json') assert jsonLoader(__module__, './model-rules.json') assert jsonLoader(__module__, '../models/sample-model.json') # test invalid relative path exception with pytest.raises(Exception): jsonLoader(__module__, './sample-model.json') # test invalid absolute path exception with pytest.raises(Exception): jsonLoader(__module__, '/model-rules.json') # test invalid file type exception with pytest.raises(Exception): jsonLoader(__module__, 'loader.py') return self
def _apply_update(root_path, service_name=''): # construct message msg_insert = 'local service' if service_name: msg_insert = 'service "%s"' % service_name # update vcs ignore import hashlib from os import path from pocketlab.methods.vcs import merge_ignores vcs_files = { 'git': { 'path': path.join(root_path, '.gitignore'), 'name': '.gitignore' }, 'mercurial': { 'path': path.join(root_path, '.hgignore'), 'name': '.hgignore' } } for key, value in vcs_files.items(): if path.exists(value['path']): old_text = open(value['path']).read() old_hash = hashlib.sha1(old_text.encode('utf-8')).hexdigest() new_text = merge_ignores(old_text, vcs_templates[key]) new_hash = hashlib.sha1(new_text.encode('utf-8')).hexdigest() if old_hash != new_hash: with open(value['path'], 'wt') as f: f.write(new_text) f.close() if verbose: print('%s file for %s updated.' % (value['name'], msg_insert)) # update lab yaml from pocketlab import __module__ from jsonmodel.loader import jsonLoader from jsonmodel.validators import jsonModel from labpack.records.settings import save_settings, load_settings config_schema = jsonLoader(__module__, 'models/lab-config.json') config_model = jsonModel(config_schema) template_config = config_model.ingest() config_path = path.join(root_path, 'lab.yaml') if path.exists(config_path): try: old_config = load_settings(config_path) template_config.update(**old_config) if old_config != template_config: from pocketlab.methods.config import compile_yaml config_text = compile_yaml(config_schema, config_path) with open(config_path, 'wt') as f: f.write(config_text) f.close() if verbose: print('lab.yaml file for %s updated.' % msg_insert) except: print('lab.yaml file for %s is corrupted. Skipped.' % msg_insert) # update setup.py setup_path = path.join(root_path, 'setup.py') if path.exists(setup_path): from pocketlab.methods.config import update_setup old_text = open(setup_path).read() old_hash = hashlib.sha1(old_text.encode('utf-8')).hexdigest() new_text = update_setup(old_text) new_hash = hashlib.sha1(new_text.encode('utf-8')).hexdigest() if old_hash != new_hash: with open(setup_path, 'wt', encoding='utf-8') as f: f.write(new_text) f.close() if verbose: print('setup.py file for %s updated.' % msg_insert)
def init(module_name='', vcs_service='', license_type='MIT', init_heroku=False,verbose=True): ''' a method to add lab framework files to the current directory :param module_name: [optional] string with name of module to create :param vcs_service: [optional] string with name of version control service :param license_type: [optional] string with name of software license type :param init_heroku: [optional] boolean to add heroku.yaml to cred folder :param verbose: [optional] boolean to toggle process messages :return: string with success exit message ''' title = 'init' # validate inputs input_fields = { 'module_name': module_name, 'vcs_service': vcs_service, 'license_type': license_type } for key, value in input_fields.items(): if value: object_title = '%s(%s=%s)' % (title, key, str(value)) fields_model.validate(value, '.%s' % key, object_title) # import dependencies from os import path # define printer submethod def _printer(file_path): if verbose: from os import path folder, file = path.split(file_path) if not folder: folder = 'working directory' else: folder = '%s folder' % folder print('"%s" created in %s.' % (file, folder)) # setup module architecture if module_name: # create vcs ignore files import re from pocketlab.methods.vcs import load_ignore vcs_regex = re.compile('#+\s+version\scontrol\s+#+') git_insert = '############# version control ################\n.hgignore\n.hg/\ndev/\ntests_dev/' hg_insert = '############# version control ################\n\\\.git/' git_path = '.gitignore' hg_path = '.hgignore' if not path.exists(git_path): git_text = load_ignore() git_text = vcs_regex.sub(git_insert, git_text) with open(git_path, 'wt') as f: f.write(git_text) f.close() _printer(git_path) if not path.exists(hg_path): hg_text = load_ignore(vcs='mercurial') hg_text = vcs_regex.sub(hg_insert, hg_text) with open(hg_path, 'wt') as f: f.write(hg_text) f.close() _printer(hg_path) # create module folder if not path.exists(module_name): from os import makedirs makedirs(module_name) _printer(module_name) # create init file init_path = path.join(module_name, '__init__.py') if not path.exists(init_path): from pocketlab.methods.config import construct_init init_text = construct_init(module_name) with open(init_path, 'wt', encoding='utf-8') as f: f.write(init_text) f.close() _printer(init_path) # create docs folder if not path.exists('docs'): from os import makedirs makedirs('docs') _printer('docs') # create mkdocs markdown file mkdocs_path = path.join('docs', 'mkdocs.md') if not path.exists(mkdocs_path): from pocketlab.methods.config import retrieve_template mkdocs_text = retrieve_template('models/mkdocs.md.txt') with open(mkdocs_path, 'wt') as f: f.write(mkdocs_text) f.close() _printer(mkdocs_path) # create index markdown file index_path = path.join('docs', 'index.md') if not path.exists(index_path): from pocketlab.methods.config import construct_index index_text = construct_index(module_name) with open(index_path, 'wt') as f: f.write(index_text) f.close() _printer(index_path) # create other root files license_type = license_type.lower() from pocketlab.methods.config import construct_changes, construct_license, construct_manifest, construct_readme, construct_mkdocs, construct_setup module_files = { 'CHANGES.rst': construct_changes(), 'LICENSE.txt': construct_license(license_type), 'MANIFEST.in': construct_manifest(module_name), 'README.rst': construct_readme(module_name=module_name), 'mkdocs.yml': construct_mkdocs(module_name), 'setup.py': construct_setup(module_name) } for key, value in module_files.items(): if not path.exists(key): with open(key, 'wt', encoding='utf-8') as f: f.write(value) f.close() _printer(key) exit_msg = 'Framework for "%s" setup in current directory.' % module_name # setup service architecture else: # determine version control service if not vcs_service: vcs_service = 'git' if path.exists('.git'): if path.isdir('.git'): vcs_service = 'git' elif path.exists('.hg'): if path.isdir('.hg'): vcs_service = 'mercurial' else: vcs_service = vcs_service.lower() # add a vcs ignore file from pocketlab.methods.vcs import load_ignore if vcs_service == 'git': vcs_path = '.gitignore' vcs_type = 'git' else: vcs_path = '.hgignore' vcs_type = 'mercurial' if not path.exists(vcs_path): file_text = load_ignore(vcs=vcs_type) with open(vcs_path, 'wt') as f: f.write(file_text) f.close() _printer(vcs_path) # add a lab config file config_path = 'lab.yaml' if not path.exists(config_path): # retrieve config model from pocketlab import __module__ from jsonmodel.loader import jsonLoader config_schema = jsonLoader(__module__, 'models/lab-config.json') # compile yaml from pocketlab.methods.config import compile_yaml config_text = compile_yaml(config_schema) # save config text with open(config_path, 'wt') as f: f.write(config_text) f.close() _printer(config_path) # add a data folder data_path = 'data' from os import makedirs if not path.exists(data_path): makedirs(data_path) _printer(data_path) # add a credential folder cred_path = 'cred' notes_path = 'notes' if not path.exists(cred_path): makedirs(cred_path) _printer(cred_path) if path.exists(notes_path): if path.isdir(notes_path): src_list = [] dst_list = [] from os import listdir from shutil import copyfile for file_name in listdir(notes_path): file_path = path.join(notes_path, file_name) if path.isfile(file_path): if file_name.find('.json') > -1 or file_name.find('.yaml') > -1 or file_name.find('.yml') > -1: src_list.append(file_path) dst_list.append(path.join(cred_path, file_name)) for i in range(len(src_list)): copyfile(src_list[i], dst_list[i]) _printer(dst_list[i]) # add config files config_map = { 'heroku.yaml': { 'toggle': init_heroku, 'schema_path': 'models/heroku-config.json' } } for key, value in config_map.items(): if value['toggle']: config_path = 'cred/%s' % key if not path.exists(config_path): from pocketlab import __module__ from jsonmodel.loader import jsonLoader from pocketlab.methods.config import compile_yaml config_schema = jsonLoader(__module__, value['schema_path']) config_text = compile_yaml(config_schema) with open(config_path, 'wt') as f: f.write(config_text) f.close() _printer(config_path) # add readme file readme_path = 'README.md' if not path.exists(readme_path): from pocketlab.methods.config import construct_readme readme_text = construct_readme(vcs_service=vcs_service) with open(readme_path, 'wt', encoding='utf-8') as f: f.write(readme_text) f.close() _printer(readme_path) exit_msg = 'Lab framework setup in current directory.' return exit_msg
def stop(service_list, verbose=True, virtualbox='default'): title = 'stop' # validate inputs if isinstance(service_list, str): if service_list: service_list = [service_list] input_fields = { 'service_list': service_list, 'verbose': verbose, 'virtualbox': virtualbox } for key, value in input_fields.items(): if value: object_title = '%s(%s=%s)' % (title, key, str(value)) fields_model.validate(value, '.%s' % key, object_title) # validate installation of docker from pocketlab.methods.docker import dockerClient docker_client = dockerClient(virtualbox_name=virtualbox, verbose=verbose) # verbosity if verbose: print('Checking service configurations...', end='', flush=True) # construct list of paths to services from pocketlab.methods.service import retrieve_services stop_list, msg_insert = retrieve_services(service_list) # construct lab list lab_list = [] # validate lab files from os import path from pocketlab.methods.validation import validate_lab from pocketlab import __module__ from jsonmodel.loader import jsonLoader from jsonmodel.validators import jsonModel lab_model = jsonModel(jsonLoader(__module__, 'models/lab-config.json')) for details in stop_list: file_path = path.join(details['path'], 'lab.yaml') service_name = details['name'] lab_details = validate_lab(lab_model, file_path, service_name) details['config'] = lab_details lab_list.append(details) # retrieve list of docker containers # validate lab config container exists # TODO resolve name discovery when namespace transmutes if verbose: print('.', end='', flush=True) # end validations if verbose: print(' done.') # stop containers exit_msg = '' for service in lab_list: container_alias = service['config']['docker_container_alias'] docker_status = docker_client.rm(container_alias) if docker_status == container_alias: service_msg = 'Container "%s" terminated.' % docker_status if len(lab_list) > 1: if verbose: print(service_msg) else: exit_msg = service_msg else: raise ValueError(docker_status) if len(lab_list) > 1: exit_msg = 'Finished terminating %s' % msg_insert return exit_msg
__author__ = 'rcj1492' __created__ = '2017.05' __license__ = 'MIT' # initialize logging import logging logging.basicConfig(level=logging.DEBUG) # retrieve schemas from pocketlab import __module__ from jsonmodel.loader import jsonLoader fields_schema = jsonLoader(__module__, 'models/lab-fields.json') cli_schema = jsonLoader(__module__, 'models/lab-cli.json') # construct fields model from pocketlab.utils import compile_model fields_model = compile_model(fields_schema, cli_schema) if __name__ == '__main__': print(fields_model.keyMap)