def _request_command(self, sys_command, pipe=False):
''' a method to handle system commands which require connectivity '''
import sys
from subprocess import Popen, PIPE, check_output
try:
if pipe:
response = Popen(sys_command, shell=True, stdout=PIPE)
for line in response.stdout:
if self.verbose:
print(line.decode('utf-8').rstrip('\n'))
sys.stdout.flush()
response.wait()
return response
else:
response = check_output(sys_command).decode('utf-8')
return response
except Exception as err:
from requests import Request
from labpack.handlers.requests import handle_requests
test_url = 'http://www.google.com'
request_object = Request(method='GET', url=test_url)
request_details = handle_requests(request_object)
if request_details['error']:
raise ConnectionError(request_details['error'])
else:
raise err
def _cli_input(self, sys_command, lambda_sequence):
''' a method to answer command line interface inputs '''
from subprocess import Popen, PIPE, CalledProcessError
try:
with Popen(sys_command,
stdin=PIPE,
stdout=PIPE,
universal_newlines=True) as p:
count = 0
for line in p.stdout:
answer = lambda_sequence[0](line)
count += 1
if not answer:
continue
print(answer, file=p.stdin)
p.stdin.flush()
except CalledProcessError:
from requests import Request
from labpack.handlers.requests import handle_requests
test_url = 'http://collectiveacuity.com'
request_object = Request(method='GET', url=test_url)
request_details = handle_requests(request_object)
raise ConnectionError(request_details['error'])
except:
raise
def __init__(self, request='', message='', errors=None, captured_error=None):
# report request attempt
self.errors = errors
text = 'Failure connecting to Google Drive API with %s request.' % request
# test connectivity
try:
import requests
requests.get('https://www.google.com')
except:
from requests import Request
from labpack.handlers.requests import handle_requests
request_object = Request(method='GET', url='https://www.google.com')
request_details = handle_requests(request_object)
text += '\n%s' % request_details['error']
# include original error message
else:
try:
if captured_error:
raise captured_error
else:
raise
except Exception as err:
text += '\n%s' % err
if message:
text += '\n%s' % message
super(DriveConnectionError, self).__init__(text)
def check_responsiveness(url, timeout=2):
import requests
from labpack.handlers.requests import handle_requests
details = {
'method': '',
'code': 0,
'url': url,
'error': '',
'json': None,
'headers': {}
}
try:
response = requests.get(url, timeout=timeout)
details['method'] = response.request.method
details['url'] = response.url
details['code'] = response.status_code
details['headers'] = response.headers
if details['code'] >= 200 and details['code'] < 300:
pass
else:
details['error'] = response.content.decode()
except Exception:
request_object = requests.Request(method='GET', url=url)
details = handle_requests(request_object)
return details
def get_kitten(api_key=''):
''' a method to get a random kitten image '''
# http://thecatapi.com/
# import dependencies
import re
import requests
from labpack.handlers.requests import handle_requests
# construct request
request_kwargs = {
'url': 'http://thecatapi.com/api/images/get',
'params': {
'format': 'xml'
}
}
if api_key:
request_kwargs['params']['api_key'] = api_key
# send request
try:
response = requests.get(**request_kwargs)
except:
request_kwargs['method'] = 'GET'
request_object = requests.Request(**request_kwargs)
return handle_requests(request_object)
# construct default response details
details = {
'method': response.request.method,
'code': response.status_code,
'url': response.url,
'error': '',
'json': None,
'headers': response.headers
}
# handle different codes
if details['code'] == 200:
xml_text = response.text
url_pattern = re.compile('<url>(.*?)</url>')
url_search = url_pattern.findall(xml_text)
if url_search:
details['json'] = {'src': url_search[0]}
else:
details['error'] = response.content.decode()
return details
def bluemix_speech2text(byte_data, file_name, auth_token):
import requests
from labpack.handlers.requests import handle_requests
request_kwargs = {
'url':
'https://stream.watsonplatform.net/speech-to-text/api/v1/recognize',
'headers': {
'Content-Type': 'audio/ogg;codecs=opus',
'X-Watson-Authorization-Token': auth_token,
'X-Watson-Learning-Opt-Out': 'true'
},
'files': {
'file': (file_name, byte_data)
}
}
# send request
try:
response = requests.post(**request_kwargs)
except:
request_kwargs['method'] = 'POST'
request_object = requests.Request(**request_kwargs)
return handle_requests(request_object)
# construct default response details
details = {
'method': response.request.method,
'code': response.status_code,
'url': response.url,
'error': '',
'json': None,
'headers': response.headers
}
# handle different codes
if details['code'] == 200:
details['json'] = response.json()
else:
details['error'] = response.content.decode()
return details
def bluemix_token(bluemix_username, bluemix_password):
import requests
from labpack.handlers.requests import handle_requests
# construct request
request_kwargs = {
'url': 'https://stream.watsonplatform.net/authorization/api/v1/token',
'params': {
'url': 'https://stream.watsonplatform.net/speech-to-text/api'
},
'auth': (bluemix_username, bluemix_password)
}
# send request
try:
response = requests.get(**request_kwargs)
except:
request_kwargs['method'] = 'GET'
request_object = requests.Request(**request_kwargs)
return handle_requests(request_object)
# construct default response details
details = {
'method': response.request.method,
'code': response.status_code,
'url': response.url,
'error': '',
'json': None,
'headers': response.headers
}
# handle different codes
if details['code'] == 200:
details['json'] = {'token': response.text}
else:
details['error'] = response.content.decode()
return details
def get_ip(source='aws'):
''' a method to get current public ip address of machine '''
if source == 'aws':
source_url = 'http://checkip.amazonaws.com/'
else:
raise Exception('get_ip currently only supports queries to aws')
import requests
try:
response = requests.get(url=source_url)
except Exception as err:
from labpack.handlers.requests import handle_requests
from requests import Request
request_object = Request(method='GET', url=source_url)
request_details = handle_requests(request_object)
raise Exception(request_details['error'])
current_ip = response.content.decode()
current_ip = current_ip.strip()
return current_ip
def describe_ip(ip_address, source='whatismyip'):
''' a method to get the details associated with an ip address '''
# determine url
if source == 'nekudo':
source_url = 'https://geoip.nekudo.com/api/%s' % ip_address
elif source == 'geoip':
source_url = 'https://freegeoip.net/json/%s' % ip_address
elif source == 'whatismyip':
# http://whatismyipaddress.com/ip-lookup
source_url = 'https://whatismyipaddress.com/ip/%s' % ip_address
else:
raise Exception('describe_ip currently only supports queries to nekudo')
# TODO incorporate geoip module and c dependencies with local database
# http://tech.marksblogg.com/ip-address-lookups-in-python.html
# send request
ip_details = {
'accuracy_radius': 0,
'asn': '',
'assignment': '',
'city': '',
'continent': '',
'country': '',
'hostname': '',
'ip': '',
'isp': '',
'latitude': 0.0,
'longitude': 0.0,
'organization': '',
'postal_code': '',
'region': '',
'timezone': '',
'type': ''
}
import requests
try:
response = requests.get(url=source_url)
except Exception as err:
from labpack.handlers.requests import handle_requests
from requests import Request
request_object = Request(method='GET', url=source_url)
request_details = handle_requests(request_object)
raise Exception(request_details['error'])
# extract response
if source == 'whatismyip':
import re
response_text = response.content.decode()
table_regex = re.compile('<table>\n<tr><th>IP.*?</table>\n<span\sstyle', re.S)
table_search = table_regex.findall(response_text)
if table_search:
table_text = table_search[0]
field_list = [ 'IP', 'Hostname', 'ISP', 'Organization', 'Type', 'ASN', 'Assignment', 'Continent', 'Country', 'State/Region', 'City', 'Latitude', 'Longitude', 'Postal Code']
for field in field_list:
field_regex = re.compile('<tr><th>%s:</th><td>(.*?)</td>' % field, re.S)
field_search = field_regex.findall(table_text)
if field_search:
ip_details[field.lower().replace(' ','_')] = field_search[0]
for field in ('longitude', 'latitude'):
if field in ip_details.keys():
coord_regex = re.compile('\-?\d+\.\d+')
coord_search = coord_regex.findall(ip_details[field])
if coord_search:
ip_details[field] = float(coord_search[0])
if 'country' in ip_details.keys():
country_regex = re.compile('([\w\s]+?)($|\s<img)')
country_search = country_regex.findall(ip_details['country'])
if country_search:
ip_details['country'] = country_search[0][0]
for field in ('type', 'assignment'):
if field in ip_details.keys():
link_regex = re.compile('>(.*?)<')
link_search = link_regex.findall(ip_details[field])
if link_search:
ip_details[field] = link_search[0]
if 'state/region' in ip_details.keys():
ip_details['region'] = ip_details['state/region']
del ip_details['state/region']
elif source == 'nekudo':
response_details = response.json()
ip_details['country'] = response_details['country']['name']
ip_details['latitude'] = response_details['location']['latitude']
ip_details['longitude'] = response_details['location']['longitude']
ip_details['accuracy_radius'] = response_details['location']['accuracy_radius']
if response_details['city']:
ip_details['city'] = response_details['city']
ip_details['ip'] = response_details['ip']
for key in response_details.keys():
if key not in ip_details.keys() and key != 'location':
ip_details[key] = response_details[key]
else:
response_details = response.json()
for field in ('city', 'ip', 'latitude', 'longitude'):
ip_details[field] = response_details[field]
ip_details['country'] = response_details['country_name']
ip_details['region'] = response_details['region_name']
ip_details['postal_code'] = response_details['zip_code']
ip_details['timezone'] = response_details['time_zone']
return ip_details
def __init__(self, magic_file=''):
''' initialization method for labMagic class
:param magic_file: [optional] string with local path to magic.mgc file
'''
title = '%s.__init__' % self.__class__.__name__
# construct class field model
from jsonmodel.validators import jsonModel
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'magic_file': magic_file
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct magic method
magic_kwargs = {
'mime': True,
'uncompress': True
}
from labpack.platforms.localhost import localhostClient
sys_name = localhostClient().os.sysname
if sys_name == 'Windows':
if not magic_file:
raise IndexError('%s(magic_file="...") is required on Windows systems.')
import os
if magic_file:
if not os.path.exists(magic_file):
raise ValueError('%s(magic_file=%s) is not a valid file path.' % (title, magic_file))
magic_kwargs['magic_file'] = magic_file
try:
# workaround for module namespace conflict
from sys import path as sys_path
sys_path.append(sys_path.pop(0))
import magic
sys_path.insert(0, sys_path.pop())
self.magic = magic.Magic(**magic_kwargs)
except:
raise Exception('\nmagiclab requires the python-magic module. try: pip install python-magic\npython-magic requires the C library libmagic. See documentation in labpack.parsing.magic.')
# construct mimetypes method
import mimetypes
self.mimetypes = mimetypes.MimeTypes()
# retrieve updates to mimetypes
mimetype_urls = self.fields.schema['mimetype_urls']
from labpack.storage.appdata import appdataClient
mime_collection = appdataClient('Mime Types')
mime_filter = mime_collection.conditional_filter([{-1:{'must_contain': ['mime.types']}}])
mime_list = mime_collection.list(mime_filter)
for key in mimetype_urls.keys():
file_path = os.path.join(mime_collection.collection_folder, key)
if key not in mime_list:
file_dir = os.path.split(file_path)[0]
if not os.path.exists(file_dir):
os.makedirs(file_dir)
import requests
try:
response = requests.get(mimetype_urls[key])
except Exception:
from labpack.handlers.requests import handle_requests
request_kwargs = {'url': mimetype_urls[key]}
response_details = handle_requests(requests.Request(**request_kwargs))
print('magiclab attempted to retrieve latest mimetype registry resource at %s but ran into this non-fatal error: %s' % (mimetype_urls[key], response_details['error']))
break
with open(file_path, 'wb') as f:
f.write(response.content)
f.close()
ext_map = mimetypes.read_mime_types(file_path)
for key, value in ext_map.items():
self.mimetypes.add_type(value, key)
def analyze(self, file_path='', file_url='', byte_data=None):
''' a method to determine the mimetype and extension of a file from its byte data
:param file_path: [optional] string with local path to file
:param file_url: [optional] string with url of file
:param byte_data: [optional] byte data from a file
:return: dictionary with file details
{
'name': 'filename.ext',
'mimetype': 'type/sub-type',
'extension': '.ext'
}
'''
title = '%s.analyze' % self.__class__.__name__
# validate inputs
input_fields = {
'file_path': file_path,
'file_url': file_url
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
if byte_data:
if not isinstance(byte_data, bytes):
raise ValueError("%s(byte_data=b'...') must be byte data" % title)
# construct empty type map
file_details = {
'name': '',
'mimetype': '',
'extension': ''
}
magic_results = None
# analyze file
if file_path:
import os
if not os.path.exists(file_path):
raise ValueError('%s(file_path=%s) is not a valid path.' % (title, file_path))
split_file = os.path.split(file_path)
file_details['name'] = split_file[0]
if len(split_file) > 1:
file_details['name'] = split_file[1]
magic_results = self.magic.from_file(file_path)
# analyze url
elif file_url:
from urllib.parse import urlsplit
url_path = urlsplit(file_url).path
path_segments = url_path.split('/')
file_details['name'] = path_segments[-1]
import requests
try:
response = requests.get(file_url)
except Exception:
from labpack.handlers.requests import handle_requests
response_details = handle_requests(requests.Request(url=file_url))
raise ValueError('%s(file_url=%s) created this error: %s' % (title, file_url, response_details['error']))
magic_results = self.magic.from_buffer(response.content)
# analyze buffer
elif byte_data:
magic_results = self.magic.from_buffer(byte_data)
else:
raise IndexError('%s(...) requires either a file_path, file_url or byte_data argument' % title)
if magic_results:
file_details['mimetype'] = magic_results
mime_results = self.mimetypes.guess_extension(magic_results)
if mime_results:
file_details['extension'] = mime_results
return file_details