def read_account(account_id='', account_email='', account_token=''):
# import dependencies
from labpack.records.settings import load_settings
from labpack.platforms.localhost import localhostClient
# construct default response
account_details = {}
account_results = []
localhost_client = localhostClient()
# search by index feature
if account_id:
id_query = [{'.file_name': {'discrete_values': ['%s.yaml' % account_id]}}]
id_filter = localhost_client.conditional_filter(id_query)
account_results = localhost_client.list(id_filter, list_root='../data/accounts/id', max_results=1)
elif account_email:
import hashlib
email_hash = hashlib.sha256(account_email.encode('utf-8')).hexdigest()
email_query = [{'.file_name': {'discrete_values': ['%s.yaml' % email_hash ]}}]
email_filter = localhost_client.conditional_filter(email_query)
account_results = localhost_client.list(email_filter, list_root='../data/accounts/email', max_results=1)
elif account_token:
token_query = [{'.file_name': {'discrete_values': ['%s.yaml' % account_token ]}}]
token_filter = localhost_client.conditional_filter(token_query)
account_results = localhost_client.list(token_filter, list_root='../data/accounts/token', max_results=1)
# return account details
if account_results:
account_details = load_settings(account_results[0])
return account_details
def construct_license(license_type='mit'):
# retrieve license text
file_path = 'models/license.%s.txt' % license_type
file_text = retrieve_template(file_path)
# retrieve username
import os
from labpack.platforms.localhost import localhostClient
localhost_client = localhostClient()
if localhost_client.os.sysname == 'Windows':
username = os.environ.get('USERNAME')
else:
home_path = os.path.abspath(localhost_client.home)
root_path, username = os.path.split(home_path)
# retrieve date
from datetime import datetime
new_date = datetime.utcnow()
new_year = str(new_date.year)
# replace terms in license
file_text = file_text.replace('2017', new_year)
file_text = file_text.replace('Collective Acuity', username)
return file_text
def __init__(self, virtualbox_name='', verbose=False):
'''
a method to initialize the dockerClient class
:param virtualbox_name: [optional] string with name of virtualbox image
:return: dockerClient object
'''
title = '%s.__init__' % self.__class__.__name__
# construct super
super(dockerClient, self).__init__()
# construct fields model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'virtualbox_name': virtualbox_name
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct properties
self.vbox = virtualbox_name
self.verbose = verbose
# construct localhost
from labpack.platforms.localhost import localhostClient
self.localhost = localhostClient()
# verbosity
if self.verbose:
print('Checking docker installation...', end='', flush=True)
# validate docker installation
self._validate_install()
if self.verbose:
print('.', end='', flush=True)
# validate virtualbox installation
self.vbox_running = self._validate_virtualbox()
if self.verbose:
print('.', end='', flush=True)
# set virtualbox variables
if self.vbox_running:
self._set_virtualbox()
if self.verbose:
print('.', end='', flush=True)
if self.verbose:
print(' done.')
def __init__(self, collection_name='', prod_name='', org_name='', root_path=''):
''' initialization method of appdata client class
:param collection_name: [optional] string with name of collection to store records
:param prod_name: [optional] string with name of application product
:param org_name: [optional] string with name of organization behind product
:param root_path: [optional] string with path to root of collections (defaults to user home)
'''
title = '%s.__init__' % self.__class__.__name__
# add localhost property to class
self.localhost = localhostClient()
# construct input validation model
self.fields = jsonModel(self._class_fields)
# validate inputs
if not collection_name:
collection_name = 'User Data'
else:
collection_name = self.fields.validate(collection_name, '.collection_name')
if not org_name:
org_name = __team__
else:
org_name = self.localhost.fields.validate(org_name, '.org_name')
if not prod_name:
prod_name = __module__
else:
prod_name = self.localhost.fields.validate(prod_name, '.prod_name')
# construct collection name
from copy import deepcopy
self.collection_name = deepcopy(collection_name)
# construct app folder
if not root_path:
self.app_folder = self.localhost.app_data(org_name=org_name, prod_name=prod_name)
else:
root_path = self.fields.validate(root_path, '.root_path')
if os.path.exists(root_path):
if not os.path.isdir(root_path):
raise ValueError('%s(root_path="%s") is an existing file.' % (title, root_path))
self.app_folder = os.path.abspath(root_path)
# validate existence of file data folder in app data (or create)
if self.localhost.os in ('Linux', 'FreeBSD', 'Solaris'):
collection_name = collection_name.replace(' ', '-').lower()
self.collection_folder = os.path.join(self.app_folder, collection_name)
self.fields.validate(self.collection_folder, '.record_key_path')
if not os.path.exists(self.collection_folder):
os.makedirs(self.collection_folder)
def recognize_text(file_path, tesseract_cmd=''):
import re
import builtins
from wand.image import Image
from PIL import Image as PI
import io
import pytesseract
image_texts = []
# define tesseract command path
if tesseract_cmd:
pytesseract.pytesseract.tesseract_cmd = tesseract_cmd
else:
from labpack.platforms.localhost import localhostClient
if localhostClient().os.sysname == 'Windows':
raise IndexError('tesseract_cmd argument required on Windows.')
# open image file
image_files = []
png_regex = re.compile('\.png$')
if not png_regex.findall(file_path):
image_files = convert_to_png(file_path)
else:
image_png = Image(filename=file_path, resolution=300)
for img in image_png.sequence:
image_page = Image(image=img)
image_files.append(image_page.make_blob('png'))
# bypass cp1252 errors in python3
original_open = open
def bin_open(filename, mode='rb'):
return original_open(filename, mode)
# process ocr with tesseract
try:
builtins.open = bin_open
for img in image_files:
bts = pytesseract.image_to_string(PI.open(io.BytesIO(img)))
text_string = str(bts, 'cp1252', 'ignore')
image_texts.append(text_string)
finally:
builtins.open = original_open
return image_texts
def delete_signatures(account_id):
# import dependencies
from labpack.platforms.localhost import localhostClient
from labpack.records.settings import remove_settings
# remove signatures
localhost_client = localhostClient()
signature_query = [{
'.file_name': {'must_contain': ['^%s' % account_id]}
}]
signature_filter = localhost_client.conditional_filter(signature_query)
signature_results = localhost_client.list(signature_filter, list_root='../data/signatures')
if signature_results:
for file_path in signature_results:
remove_settings(file_path)
return signature_results
def construct_init(module_name):
'''
a method to create the text for a __init__.py file for a new module
:param module_name: string with name of module to create
:return: string with text for init file
'''
# retrieve username
import os
from labpack.platforms.localhost import localhostClient
localhost_client = localhostClient()
if localhost_client.os.sysname == 'Windows':
username = os.environ.get('USERNAME')
else:
home_path = os.path.abspath(localhost_client.home)
root_path, username = os.path.split(home_path)
# retrieve date
from datetime import datetime
new_date = datetime.utcnow()
new_month = str(new_date.month)
if len(new_month) == 1:
new_month = '0%s' % new_month
date_string = '%s.%s' % (str(new_date.year), new_month)
# construct init text
init_text = "''' A Brand New Python Module '''"
init_text += "__author__ = '%s'\n" % username
init_text += "__created__ = '%s'\n" % date_string
init_text += "__module__ = '%s'\n" % module_name
init_text += "__version__ = '0.1'\n"
init_text += "__license__ = 'MIT' # BSD, ALv2, GPLv3+, LGPLv3+, ©%s Collective Acuity\n" % str(new_date.year)
init_text += "__team__ = 'Collective Acuity'\n"
init_text += "__email__ = '*****@*****.**'\n"
init_text += "__url__ = 'https://github.com/collectiveacuity/%s'\n" % module_name
init_text += "__description__ = 'A Brand New Python Module'\n"
return init_text
def _create_folder(self):
''' a helper method for creating a temporary audio clip folder '''
# import dependencies
import os
from labpack.platforms.localhost import localhostClient
from labpack.records.id import labID
# create folder in user app data
record_id = labID()
collection_name = 'Watson Speech2Text'
localhost_client = localhostClient()
app_folder = localhost_client.app_data(org_name=__team__, prod_name=__module__)
if localhost_client.os in ('Linux', 'FreeBSD', 'Solaris'):
collection_name = collection_name.replace(' ', '-').lower()
collection_folder = os.path.join(app_folder, collection_name)
clip_folder = os.path.join(collection_folder, record_id.id24)
if not os.path.exists(clip_folder):
os.makedirs(clip_folder)
return clip_folder
def __init__(self,
account_email,
account_password,
app_subdomain,
verbose=False):
''' a method to initialize the herokuClient class '''
title = '%s.__init__' % self.__class__.__name__
# construct fields model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'account_email': account_email,
'account_password': account_password,
'app_subdomain': app_subdomain
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
self.email = account_email
self.password = account_password
self.subdomain = app_subdomain
# construct class properties
self.verbose = verbose
# construct localhost
from labpack.platforms.localhost import localhostClient
self.localhost = localhostClient()
# validate installation
self._validate_install()
# validate access
self._validate_access()
def __init__(self, virtualbox_name='', verbose=False):
'''
a method to initialize the dockerClient class
:param virtualbox_name: [optional] string with name of virtualbox image
:return: dockerClient object
'''
# construct vbox property
self.vbox = virtualbox_name
self.verbose = verbose
# construct localhost
from labpack.platforms.localhost import localhostClient
self.localhost = localhostClient()
# verbosity
if self.verbose:
print('Checking docker installation...', end='', flush=True)
# validate docker installation
self._validate_install()
if self.verbose:
print('.', end='', flush=True)
# validate virtualbox installation
box_running = self._validate_virtualbox()
if self.verbose:
print('.', end='', flush=True)
# set virtualbox variables
if box_running:
self._set_virtualbox()
if self.verbose:
print('.', end='', flush=True)
if self.verbose:
print(' done.')
def __init__(self, account_email, auth_token, verbose=True):
''' a method to initialize the herokuClient class '''
title = '%s.__init__' % self.__class__.__name__
# initialize super
super(herokuClient, self).__init__(verbose=verbose)
# construct fields model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'account_email': account_email,
'auth_token': auth_token
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct properties
self.email = account_email
self.token = auth_token
self.subdomain = ''
self.apps = []
# construct localhost
from labpack.platforms.localhost import localhostClient
self.localhost = localhostClient()
# validate installation
self._validate_install()
# validate access
self._validate_login()
def home(service_name, print_path=False, service_path='', overwrite=False):
'''
a method to manage the local path information for a service
:param service_name: string with name of service to add to registry
:param print_path: [optional] boolean to retrieve local path of service from registry
:param service_path: [optional] string with path to service root
:param overwrite: [optional] boolean to overwrite existing service registration
:return: string with local path to service
'''
title = 'home'
# validate inputs
input_map = {'service_name': service_name, 'service_path': service_path}
for key, value in input_map.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
fields_model.validate(value, '.%s' % key, object_title)
# validate requirements
# TODO boolean algebra method to check not both inputs
# resolve print path request
if print_path:
# retrieve service root
from pocketlab.methods.service import retrieve_service_root
command_context = 'Try running "lab home %s" first from its root.' % service_name
service_root = retrieve_service_root(service_name, command_context)
# return root path to bash command
import sys
exit_msg = 'Transport to "%s" underway.;%s' % (service_name,
service_root)
print(exit_msg)
sys.exit()
# resolve service request
# validate existence of home alias
from os import path
from labpack.platforms.localhost import localhostClient
localhost_client = localhostClient()
home_alias = "alias home='function _home(){ lab_output=\"$(lab home --print $1)\"; IFS=\";\" read -ra LINES <<< \"$lab_output\"; echo \"${LINES[0]}\"; cd \"${LINES[1]}\"; };_home'"
config_list = [localhost_client.bash_config, localhost_client.sh_config]
for i in range(len(config_list)):
if config_list[i]:
if not path.exists(config_list[i]):
with open(config_list[i], 'wt') as f:
f.write('# alias for pocketlab home command\n')
f.write(home_alias)
f.close()
else:
import re
home_pattern = re.compile('alias home\=')
lab_pattern = re.compile(
'alias home\=\'function _home\(\)\{\slab_output')
home_match = False
lab_match = False
with open(config_list[i], 'rt') as f:
for line in f:
line = line.partition('#')[0]
line = line.rstrip()
if home_pattern.findall(line):
home_match = True
if lab_pattern.findall(line):
lab_match = True
if not home_match:
with open(config_list[i], 'a') as f:
f.write('\n\n# alias for pocketlab home command\n')
f.write(home_alias)
f.close()
elif not lab_match:
raise ValueError(
'the "home" alias is being used by another program.')
else:
pass
# TODO allow declaration of different alias
# TODO check system path for home command
# construct registry client
from pocketlab import __module__
from labpack.storage.appdata import appdataClient
registry_client = appdataClient(collection_name='Registry Data',
prod_name=__module__)
# validate service name is not already in registry
file_name = '%s.yaml' % service_name
filter_function = registry_client.conditional_filter([{
0: {
'discrete_values': [file_name]
}
}])
service_list = registry_client.list(filter_function=filter_function)
if file_name in service_list:
if not overwrite:
suggest_msg = 'Add -f to overwrite.'
raise ValueError('"%s" already exists in the registry. %s' %
(service_name, suggest_msg))
# add service to registry
service_root = './'
if service_path:
if not path.exists(service_path):
raise ValueError('"%s" is not a valid path.' % service_path)
elif not path.isdir(service_path):
raise ValueError('"%s" is not a valid directory.' % service_path)
service_root = service_path
file_details = {
'service_name': service_name,
'service_root': path.abspath(service_root)
}
registry_client.create(file_name, file_details)
exit_msg = '"%s" added to registry. To return to workdir, run "home %s"' % (
service_name, service_name)
return exit_msg
def __init__(self, instance_id, pem_file, access_id, secret_key, region_name, owner_id, user_name, login_name='', verbose=True):
'''
a method for initializing the SSH connection parameters to the EC2 instance
:param instance_id: string with AWS id of instance
:param pem_file: string with path to keypair pem file
:param access_id: string with access_key_id from aws IAM user setup
:param secret_key: string with secret_access_key from aws IAM user setup
:param region_name: string with name of aws region
:param owner_id: string with aws account id
:param user_name: string with name of user access keys are assigned to
:param login_name: [optional] string with name of login user
:param verbose: boolean to enable process messages
'''
title = '%s.__init__' % self.__class__.__name__
# initialize model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# construct localhost client
from labpack.platforms.localhost import localhostClient
self.localhost = localhostClient()
# validate credentials and construct ec2 method
from labpack.platforms.aws.ec2 import ec2Client
self.ec2 = ec2Client(access_id, secret_key, region_name, owner_id, user_name, verbose)
# validate inputs
input_fields = {
'instance_id': instance_id,
'pem_file': pem_file
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.ec2.fields.validate(value, '.%s' % key, object_title)
# construct class properties
self.instance_id = instance_id
# verify pem file exists
from os import path
if not path.exists(pem_file):
raise Exception('%s is not a valid path.' % pem_file)
self.pem_file = path.abspath(pem_file)
# verify user has privileges
try:
self.ec2.iam.printer_on = False
self.ec2.list_keypairs()
except AWSConnectionError as err:
if str(err).find('Could not connect') > -1:
raise
raise AWSConnectionError(title, 'You must have privileges to access EC2 to use sshClient')
# verify instance exists
instance_list = self.ec2.list_instances()
if instance_id not in instance_list:
raise Exception('%s does not exist in this region or permission scope.' % instance_id)
# verify instance has public ip
instance_details = self.ec2.read_instance(instance_id)
if not instance_details['public_ip_address']:
raise Exception('%s requires a public IP address to access through ssh.' % instance_id)
self.instance_ip = instance_details['public_ip_address']
# retrieve login name from tag
self.login_name = ''
input_fields = {
'login_name': login_name
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
self.login_name = login_name
if not self.login_name:
for tag in instance_details['tags']:
if tag['key'] == 'UserName':
self.login_name = tag['value']
if not self.login_name:
raise Exception('SSH access to %s requires a login_name argument or UserName tag' % instance_id)
# verify local and remote pem file names match
from os import path
pem_absolute = path.abspath(pem_file)
pem_root, pem_ext = path.splitext(pem_absolute)
pem_path, pem_name = path.split(pem_root)
if not instance_details['key_name'] == pem_name:
raise Exception('%s does not match name of keypair %s for instance %s.' % (pem_name, instance_details['keypair'], instance_id))
# verify instance is ready
self.ec2.check_instance_status(instance_id)
# verify security group allows ssh
group_list = []
for group in instance_details['security_groups']:
group_list.append(group['group_id'])
if not group_list:
raise Exception('SSH access to %s requires a security group attached to instance.' % instance_id)
port_22_list = []
for group_id in group_list:
group_details = self.ec2.read_security_group(group_id)
for permission in group_details['ip_permissions']:
if permission['from_port'] == 22:
port_22_list.extend(permission['ip_ranges'])
if not port_22_list:
raise Exception('SSH access to %s requires a security group with inbound permissions for port 22.' % instance_id)
from labpack.records.ip import get_ip
current_ip = get_ip()
ssh_access = False
for ip_range in port_22_list:
if ip_range['cidr_ip'].find('0.0.0.0/0') > -1 or ip_range['cidr_ip'].find(current_ip) > -1:
ssh_access = True
break
if not ssh_access:
raise Exception('SSH access to %s requires your IP %s to be added to port 22 on its security group.' % (instance_id, current_ip))
# verify pem file has access
try:
self.script('ls -a')
except:
raise AWSConnectionError(title, '%s does not have access to instance %s.' % (pem_name, instance_id))
# verify scp
self.scp = False
# turn printer back on
self.ec2.iam.printer_on = True
def __init__(self, magic_file=''):
''' initialization method for labMagic class
:param magic_file: [optional] string with local path to magic.mgc file
'''
title = '%s.__init__' % self.__class__.__name__
# construct class field model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'magic_file': magic_file
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct magic method
magic_kwargs = {
'mime': True,
'uncompress': True
}
from labpack.platforms.localhost import localhostClient
sys_name = localhostClient().os.sysname
if sys_name == 'Windows':
if not magic_file:
raise IndexError('%s(magic_file="...") is required on Windows systems.')
import os
if magic_file:
if not os.path.exists(magic_file):
raise ValueError('%s(magic_file=%s) is not a valid file path.' % (title, magic_file))
magic_kwargs['magic_file'] = magic_file
try:
# workaround for module namespace conflict
from sys import path as sys_path
sys_path.append(sys_path.pop(0))
import magic
sys_path.insert(0, sys_path.pop())
self.magic = magic.Magic(**magic_kwargs)
except:
raise Exception('\nmagiclab requires the python-magic module. try: pip install python-magic\npython-magic requires the C library libmagic. See documentation in labpack.parsing.magic.')
# construct mimetypes method
import mimetypes
self.mimetypes = mimetypes.MimeTypes()
# retrieve updates to mimetypes
mimetype_urls = self.fields.schema['mimetype_urls']
from labpack.storage.appdata import appdataClient
mime_collection = appdataClient('Mime Types')
mime_filter = mime_collection.conditional_filter([{-1:{'must_contain': ['mime.types']}}])
mime_list = mime_collection.list(mime_filter)
for key in mimetype_urls.keys():
file_path = os.path.join(mime_collection.collection_folder, key)
if key not in mime_list:
file_dir = os.path.split(file_path)[0]
if not os.path.exists(file_dir):
os.makedirs(file_dir)
import requests
try:
response = requests.get(mimetype_urls[key])
except Exception:
from labpack.handlers.requests import handle_requests
request_kwargs = {'url': mimetype_urls[key]}
response_details = handle_requests(requests.Request(**request_kwargs))
print('magiclab attempted to retrieve latest mimetype registry resource at %s but ran into this non-fatal error: %s' % (mimetype_urls[key], response_details['error']))
break
with open(file_path, 'wb') as f:
f.write(response.content)
f.close()
ext_map = mimetypes.read_mime_types(file_path)
for key, value in ext_map.items():
self.mimetypes.add_type(value, key)
__author__ = 'rcj1492'
__created__ = '2016.11'
__license__ = 'MIT'
from labpack.parsing.magic import labMagic
if __name__ == '__main__':
from labpack.platforms.localhost import localhostClient
localhost_client = localhostClient()
file_path = '../data/test_photo.diy'
file_url = 'https://pbs.twimg.com/profile_images/479475632158408704/Zelyz-xr_400x400.png'
byte_data = open('../data/test_pdf.pde', 'rb').read()
if localhost_client.os.sysname in ('Windows'):
lab_magic = labMagic('../data/magic.mgc')
else:
lab_magic = labMagic()
analysis = lab_magic.analyze(file_path)
assert analysis['extension'] == '.png'
analysis = lab_magic.analyze(file_url=file_url)
assert analysis['name'] == 'Zelyz-xr_400x400.png'
analysis = lab_magic.analyze(byte_data=byte_data)
assert analysis['mimetype'] == 'application/pdf'
not_a_file = 'happy'.encode('utf-8')
analysis = lab_magic.analyze(byte_data=not_a_file)
assert analysis['mimetype'] == 'text/plain'