def _request_command(self, sys_command, pipe=False): ''' a method to handle system commands which require connectivity ''' import sys from subprocess import Popen, PIPE, check_output try: if pipe: response = Popen(sys_command, shell=True, stdout=PIPE) for line in response.stdout: if self.verbose: print(line.decode('utf-8').rstrip('\n')) sys.stdout.flush() response.wait() return response else: response = check_output(sys_command).decode('utf-8') return response except Exception as err: from requests import Request from labpack.handlers.requests import handle_requests test_url = 'http://www.google.com' request_object = Request(method='GET', url=test_url) request_details = handle_requests(request_object) if request_details['error']: raise ConnectionError(request_details['error']) else: raise err
def _cli_input(self, sys_command, lambda_sequence): ''' a method to answer command line interface inputs ''' from subprocess import Popen, PIPE, CalledProcessError try: with Popen(sys_command, stdin=PIPE, stdout=PIPE, universal_newlines=True) as p: count = 0 for line in p.stdout: answer = lambda_sequence[0](line) count += 1 if not answer: continue print(answer, file=p.stdin) p.stdin.flush() except CalledProcessError: from requests import Request from labpack.handlers.requests import handle_requests test_url = 'http://collectiveacuity.com' request_object = Request(method='GET', url=test_url) request_details = handle_requests(request_object) raise ConnectionError(request_details['error']) except: raise
def __init__(self, request='', message='', errors=None, captured_error=None): # report request attempt self.errors = errors text = 'Failure connecting to Google Drive API with %s request.' % request # test connectivity try: import requests requests.get('https://www.google.com') except: from requests import Request from labpack.handlers.requests import handle_requests request_object = Request(method='GET', url='https://www.google.com') request_details = handle_requests(request_object) text += '\n%s' % request_details['error'] # include original error message else: try: if captured_error: raise captured_error else: raise except Exception as err: text += '\n%s' % err if message: text += '\n%s' % message super(DriveConnectionError, self).__init__(text)
def check_responsiveness(url, timeout=2): import requests from labpack.handlers.requests import handle_requests details = { 'method': '', 'code': 0, 'url': url, 'error': '', 'json': None, 'headers': {} } try: response = requests.get(url, timeout=timeout) details['method'] = response.request.method details['url'] = response.url details['code'] = response.status_code details['headers'] = response.headers if details['code'] >= 200 and details['code'] < 300: pass else: details['error'] = response.content.decode() except Exception: request_object = requests.Request(method='GET', url=url) details = handle_requests(request_object) return details
def get_kitten(api_key=''): ''' a method to get a random kitten image ''' # http://thecatapi.com/ # import dependencies import re import requests from labpack.handlers.requests import handle_requests # construct request request_kwargs = { 'url': 'http://thecatapi.com/api/images/get', 'params': { 'format': 'xml' } } if api_key: request_kwargs['params']['api_key'] = api_key # send request try: response = requests.get(**request_kwargs) except: request_kwargs['method'] = 'GET' request_object = requests.Request(**request_kwargs) return handle_requests(request_object) # construct default response details details = { 'method': response.request.method, 'code': response.status_code, 'url': response.url, 'error': '', 'json': None, 'headers': response.headers } # handle different codes if details['code'] == 200: xml_text = response.text url_pattern = re.compile('<url>(.*?)</url>') url_search = url_pattern.findall(xml_text) if url_search: details['json'] = {'src': url_search[0]} else: details['error'] = response.content.decode() return details
def bluemix_speech2text(byte_data, file_name, auth_token): import requests from labpack.handlers.requests import handle_requests request_kwargs = { 'url': 'https://stream.watsonplatform.net/speech-to-text/api/v1/recognize', 'headers': { 'Content-Type': 'audio/ogg;codecs=opus', 'X-Watson-Authorization-Token': auth_token, 'X-Watson-Learning-Opt-Out': 'true' }, 'files': { 'file': (file_name, byte_data) } } # send request try: response = requests.post(**request_kwargs) except: request_kwargs['method'] = 'POST' request_object = requests.Request(**request_kwargs) return handle_requests(request_object) # construct default response details details = { 'method': response.request.method, 'code': response.status_code, 'url': response.url, 'error': '', 'json': None, 'headers': response.headers } # handle different codes if details['code'] == 200: details['json'] = response.json() else: details['error'] = response.content.decode() return details
def bluemix_token(bluemix_username, bluemix_password): import requests from labpack.handlers.requests import handle_requests # construct request request_kwargs = { 'url': 'https://stream.watsonplatform.net/authorization/api/v1/token', 'params': { 'url': 'https://stream.watsonplatform.net/speech-to-text/api' }, 'auth': (bluemix_username, bluemix_password) } # send request try: response = requests.get(**request_kwargs) except: request_kwargs['method'] = 'GET' request_object = requests.Request(**request_kwargs) return handle_requests(request_object) # construct default response details details = { 'method': response.request.method, 'code': response.status_code, 'url': response.url, 'error': '', 'json': None, 'headers': response.headers } # handle different codes if details['code'] == 200: details['json'] = {'token': response.text} else: details['error'] = response.content.decode() return details
def get_ip(source='aws'): ''' a method to get current public ip address of machine ''' if source == 'aws': source_url = 'http://checkip.amazonaws.com/' else: raise Exception('get_ip currently only supports queries to aws') import requests try: response = requests.get(url=source_url) except Exception as err: from labpack.handlers.requests import handle_requests from requests import Request request_object = Request(method='GET', url=source_url) request_details = handle_requests(request_object) raise Exception(request_details['error']) current_ip = response.content.decode() current_ip = current_ip.strip() return current_ip
def describe_ip(ip_address, source='whatismyip'): ''' a method to get the details associated with an ip address ''' # determine url if source == 'nekudo': source_url = 'https://geoip.nekudo.com/api/%s' % ip_address elif source == 'geoip': source_url = 'https://freegeoip.net/json/%s' % ip_address elif source == 'whatismyip': # http://whatismyipaddress.com/ip-lookup source_url = 'https://whatismyipaddress.com/ip/%s' % ip_address else: raise Exception('describe_ip currently only supports queries to nekudo') # TODO incorporate geoip module and c dependencies with local database # http://tech.marksblogg.com/ip-address-lookups-in-python.html # send request ip_details = { 'accuracy_radius': 0, 'asn': '', 'assignment': '', 'city': '', 'continent': '', 'country': '', 'hostname': '', 'ip': '', 'isp': '', 'latitude': 0.0, 'longitude': 0.0, 'organization': '', 'postal_code': '', 'region': '', 'timezone': '', 'type': '' } import requests try: response = requests.get(url=source_url) except Exception as err: from labpack.handlers.requests import handle_requests from requests import Request request_object = Request(method='GET', url=source_url) request_details = handle_requests(request_object) raise Exception(request_details['error']) # extract response if source == 'whatismyip': import re response_text = response.content.decode() table_regex = re.compile('<table>\n<tr><th>IP.*?</table>\n<span\sstyle', re.S) table_search = table_regex.findall(response_text) if table_search: table_text = table_search[0] field_list = [ 'IP', 'Hostname', 'ISP', 'Organization', 'Type', 'ASN', 'Assignment', 'Continent', 'Country', 'State/Region', 'City', 'Latitude', 'Longitude', 'Postal Code'] for field in field_list: field_regex = re.compile('<tr><th>%s:</th><td>(.*?)</td>' % field, re.S) field_search = field_regex.findall(table_text) if field_search: ip_details[field.lower().replace(' ','_')] = field_search[0] for field in ('longitude', 'latitude'): if field in ip_details.keys(): coord_regex = re.compile('\-?\d+\.\d+') coord_search = coord_regex.findall(ip_details[field]) if coord_search: ip_details[field] = float(coord_search[0]) if 'country' in ip_details.keys(): country_regex = re.compile('([\w\s]+?)($|\s<img)') country_search = country_regex.findall(ip_details['country']) if country_search: ip_details['country'] = country_search[0][0] for field in ('type', 'assignment'): if field in ip_details.keys(): link_regex = re.compile('>(.*?)<') link_search = link_regex.findall(ip_details[field]) if link_search: ip_details[field] = link_search[0] if 'state/region' in ip_details.keys(): ip_details['region'] = ip_details['state/region'] del ip_details['state/region'] elif source == 'nekudo': response_details = response.json() ip_details['country'] = response_details['country']['name'] ip_details['latitude'] = response_details['location']['latitude'] ip_details['longitude'] = response_details['location']['longitude'] ip_details['accuracy_radius'] = response_details['location']['accuracy_radius'] if response_details['city']: ip_details['city'] = response_details['city'] ip_details['ip'] = response_details['ip'] for key in response_details.keys(): if key not in ip_details.keys() and key != 'location': ip_details[key] = response_details[key] else: response_details = response.json() for field in ('city', 'ip', 'latitude', 'longitude'): ip_details[field] = response_details[field] ip_details['country'] = response_details['country_name'] ip_details['region'] = response_details['region_name'] ip_details['postal_code'] = response_details['zip_code'] ip_details['timezone'] = response_details['time_zone'] return ip_details
def __init__(self, magic_file=''): ''' initialization method for labMagic class :param magic_file: [optional] string with local path to magic.mgc file ''' title = '%s.__init__' % self.__class__.__name__ # construct class field model from jsonmodel.validators import jsonModel self.fields = jsonModel(self._class_fields) # validate inputs input_fields = { 'magic_file': magic_file } for key, value in input_fields.items(): if value: object_title = '%s(%s=%s)' % (title, key, str(value)) self.fields.validate(value, '.%s' % key, object_title) # construct magic method magic_kwargs = { 'mime': True, 'uncompress': True } from labpack.platforms.localhost import localhostClient sys_name = localhostClient().os.sysname if sys_name == 'Windows': if not magic_file: raise IndexError('%s(magic_file="...") is required on Windows systems.') import os if magic_file: if not os.path.exists(magic_file): raise ValueError('%s(magic_file=%s) is not a valid file path.' % (title, magic_file)) magic_kwargs['magic_file'] = magic_file try: # workaround for module namespace conflict from sys import path as sys_path sys_path.append(sys_path.pop(0)) import magic sys_path.insert(0, sys_path.pop()) self.magic = magic.Magic(**magic_kwargs) except: raise Exception('\nmagiclab requires the python-magic module. try: pip install python-magic\npython-magic requires the C library libmagic. See documentation in labpack.parsing.magic.') # construct mimetypes method import mimetypes self.mimetypes = mimetypes.MimeTypes() # retrieve updates to mimetypes mimetype_urls = self.fields.schema['mimetype_urls'] from labpack.storage.appdata import appdataClient mime_collection = appdataClient('Mime Types') mime_filter = mime_collection.conditional_filter([{-1:{'must_contain': ['mime.types']}}]) mime_list = mime_collection.list(mime_filter) for key in mimetype_urls.keys(): file_path = os.path.join(mime_collection.collection_folder, key) if key not in mime_list: file_dir = os.path.split(file_path)[0] if not os.path.exists(file_dir): os.makedirs(file_dir) import requests try: response = requests.get(mimetype_urls[key]) except Exception: from labpack.handlers.requests import handle_requests request_kwargs = {'url': mimetype_urls[key]} response_details = handle_requests(requests.Request(**request_kwargs)) print('magiclab attempted to retrieve latest mimetype registry resource at %s but ran into this non-fatal error: %s' % (mimetype_urls[key], response_details['error'])) break with open(file_path, 'wb') as f: f.write(response.content) f.close() ext_map = mimetypes.read_mime_types(file_path) for key, value in ext_map.items(): self.mimetypes.add_type(value, key)
def analyze(self, file_path='', file_url='', byte_data=None): ''' a method to determine the mimetype and extension of a file from its byte data :param file_path: [optional] string with local path to file :param file_url: [optional] string with url of file :param byte_data: [optional] byte data from a file :return: dictionary with file details { 'name': 'filename.ext', 'mimetype': 'type/sub-type', 'extension': '.ext' } ''' title = '%s.analyze' % self.__class__.__name__ # validate inputs input_fields = { 'file_path': file_path, 'file_url': file_url } for key, value in input_fields.items(): if value: object_title = '%s(%s=%s)' % (title, key, str(value)) self.fields.validate(value, '.%s' % key, object_title) if byte_data: if not isinstance(byte_data, bytes): raise ValueError("%s(byte_data=b'...') must be byte data" % title) # construct empty type map file_details = { 'name': '', 'mimetype': '', 'extension': '' } magic_results = None # analyze file if file_path: import os if not os.path.exists(file_path): raise ValueError('%s(file_path=%s) is not a valid path.' % (title, file_path)) split_file = os.path.split(file_path) file_details['name'] = split_file[0] if len(split_file) > 1: file_details['name'] = split_file[1] magic_results = self.magic.from_file(file_path) # analyze url elif file_url: from urllib.parse import urlsplit url_path = urlsplit(file_url).path path_segments = url_path.split('/') file_details['name'] = path_segments[-1] import requests try: response = requests.get(file_url) except Exception: from labpack.handlers.requests import handle_requests response_details = handle_requests(requests.Request(url=file_url)) raise ValueError('%s(file_url=%s) created this error: %s' % (title, file_url, response_details['error'])) magic_results = self.magic.from_buffer(response.content) # analyze buffer elif byte_data: magic_results = self.magic.from_buffer(byte_data) else: raise IndexError('%s(...) requires either a file_path, file_url or byte_data argument' % title) if magic_results: file_details['mimetype'] = magic_results mime_results = self.mimetypes.guess_extension(magic_results) if mime_results: file_details['extension'] = mime_results return file_details