def test_generate_file_names(self): url_generator = UrlGenerator(UrlGeneratorTest.url, UrlGeneratorTest.words) urls = [url for url in url_generator.generate_file_names()] test_str_a = 'www.foo.com/thislineshouldbeincluded.php' test_str_b = 'www.foo.com/thisaswell.php' self.assertEqual(2, len(urls)) self.assertTrue(test_str_a in urls) self.assertTrue(test_str_b in urls)
def get_related_artists(self, artist): url_generator = UrlGenerator() resp = req.request(method='GET', url=url_generator.generate(artist)) soup = BeautifulSoup(resp.text, 'html.parser') artists = [] for link in soup.find_all("a", class_="S"): artists.append(link.get_text()) return artists
def __init__(self, wsk_auth, apihost='localhost', verify=True): """See: https://console.ng.bluemix.net/openwhisk/learn/cli Your ~100 char auth can be found at that URL or by doing `wsk property get`""" self.session = None # print(get_wsk_auth()) # If wsk_auth token was not provided, then look it up in os.environ... wsk_auth = wsk_auth try: wsk_auth = tuple(wsk_auth.split(':')) except: print('Invocation error: auth must be provided or environment ' 'variable $OPENWHISK_APIHOST must be defined and must ' 'contain a colon (":"). See: `wsk property get`\n') raise self.session = requests.Session() # speeds up repeated requests self.session.verify = verify # verify SSL certs (bool) self.session.auth = wsk_auth # uses our auth token for all calls self.gen = UrlGenerator(apihost)
def test_constructor_params(): url_generator = UrlGenerator(Path(__file__).parent / 'test.json', host='example.com', port='666') assert 'http://example.com:666/under-construction' == url_generator.get_url( 'fully_parametric_site', path='under-construction') assert 'http://cool.com:666/baby' == url_generator.get_url( 'fully_parametric_site', host='cool.com', path='baby') try: url_generator.get_url('fully_parametric_site') assert False except UrlGeneratorException: assert True
def __init__(self, source=''): self.source = source self.headers = HeaderGenerator.get_headers() self.url = UrlGenerator.get_url(source=source, search='')
'activations?docs=True&name=n+A+m+E', 'activations?skip=1&limit=2', 'activations?upto=99999999&since=77', 'activations/activation_id', 'activations/activation_id/logs', 'activations/activation_id/result', 'packages', 'packages?public=True', 'packages?skip=1&limit=2', 'packages/n+A+m+E', 'packages/n+A+m+E?overwrite=True', 'rules', 'rules?skip=1&limit=2', 'rules/n+A+m+E?state=disabled', 'rules/n+A+m+E?overwrite=True', 'triggers', 'triggers?skip=1&limit=2', 'triggers/n+A+m+E?state=disabled', 'triggers/n+A+m+E?overwrite=True') ] # print(urls0) gen = UrlGenerator() urls1 = [ gen.url_action(), gen.url_action(skip=1, limit=2), gen.url_action('n A m E'), gen.url_action('n A m E', blocking=False), gen.url_action('n A m E', overwrite=True), gen.url_activation(), gen.url_activation( docs=True, name='n A m E', ), gen.url_activation(skip=1, limit=2), gen.url_activation(upto=99999999, since=77), gen.url_activation('activation_id'), gen.url_activation('activation_id', 'logs'),
class OpenWhisk(object): """https://console.ng.bluemix.net/docs/openwhisk/openwhisk_reference.html https://console.ng.bluemix.net/apidocs/98-ibm-bluemix-openwhisk""" def __init__(self, wsk_auth, apihost='localhost', verify=True): """See: https://console.ng.bluemix.net/openwhisk/learn/cli Your ~100 char auth can be found at that URL or by doing `wsk property get`""" self.session = None # print(get_wsk_auth()) # If wsk_auth token was not provided, then look it up in os.environ... wsk_auth = wsk_auth try: wsk_auth = tuple(wsk_auth.split(':')) except: print('Invocation error: auth must be provided or environment ' 'variable $OPENWHISK_APIHOST must be defined and must ' 'contain a colon (":"). See: `wsk property get`\n') raise self.session = requests.Session() # speeds up repeated requests self.session.verify = verify # verify SSL certs (bool) self.session.auth = wsk_auth # uses our auth token for all calls self.gen = UrlGenerator(apihost) def __del__(self): self.session.close() # Dynamic URLs that change as self.package changes @property def package(self): """Append '/packages/{package}' to URLs only if self.package is set.""" return self.gen.package @package.setter def package(self, package_name): self.gen.package = package_name # Actions ================================================================= """Actions""" @property def actions(self): """Returns a list of all actions in url_current_package.""" return self.actions_list() @property def action_names(self): """Returns a sorted list of the names of all actions.""" return sorted( action.get('name', 'Can\'t get action name') for action in self.actions) def action_create(self, filename, action_name, runtime='python3', *args, **kwargs): """Uploads contents of the specified file to the specified action.""" # Read the file into a string with open(filename, 'rb') as in_file: code = in_file.read() if filename.lower().split('.')[-1] == 'zip': code = base64.b64encode(code) code = code.decode('utf-8') # TODO: Support more languages beyond Python or NodeJS image = 'bspar/openwhisk-runtime-python:{}-latest'.format(runtime) payload = {'exec': {'kind': 'blackbox', 'code': code, 'image': image}} url = self.gen.url_action(action_name, *args, **kwargs) return self._put(url, payload).json() def sequence_create(self, sequence_name, action_names, *args, **kwargs): payload = {'exec': {'kind': 'sequence', 'components': action_names}} url = self.gen.url_action(sequence_name, *args, **kwargs) return self._put(url, payload).json() def action_delete(self, action_name, *args, **kwargs): """Deletes the specified action.""" url = self.gen.url_action(action_name, *args, **kwargs) return self._delete(url).json() def action_invoke(self, action_name, *args, **kwargs): """Invokes the specified action in blocking mode.""" '''url = (self.url_actions + '/' + action_name + '?blocking=true&result=false')''' payload = kwargs.pop('payload') if 'payload' in kwargs else {} url = self.gen.url_action(action_name, *args, **kwargs) return self._post(url, payload).json() def action_get(self, action_name, *args, **kwargs): return self.actions_list(action_name, *args, **kwargs) def actions_list(self, *args, **kwargs): """Lists the actions defined in openwhisk.""" url = self.gen.url_action(*args, **kwargs) return self._get(url).json() # Activations ============================================================= @property def activations(self): """Returns a sorted list of the names of all activation.""" # TODO: remove set() return sorted( set( activation.get('name') for activation in self.activations_list().json())) @property def activation_counts(self): """Returns dict of how many times current actions have been invoked.""" return collections.Counter( activation.get('name') for activation in self.activations_list().json()) @property def activation_ids(self): """Returns a sorted list of the ids of all activation.""" return sorted(activation['activationId'] for activation in self.activations_list().json()) def activation_info(self, activation_id): """Returns info on the activation.""" url = self.gen.url_activation(activation_id) return self._get(url).json() ''' def activation_logs(self, activation_id): """Returns the logs for a given activation id.""" return self.get(self.url_activations + '/' + activation_id + '/logs') ''' def activation_results(self, activation_id): """Returns a sorted list of the ids of all activation.""" url = self.gen.url_activation(activation_id, 'result') return self._get(url).json() def activations_list(self): """Lists the activations defined in openwhisk.""" return self._get(self.gen.url_activation()) ''' # Namespaces ============================================================== @property def namespaces(self): # TODO: namespaces or namespace_names """Returns a sorted list of the names of all namespaces.""" return self.namespaces_list().json() def namespaces_list(self): """Lists the namespaces defined in openwhisk.""" return self._get(self.url_namespaces) ''' # Packages ================================================================ @property def packages(self): """Returns a sorted list of the names of all packages.""" return sorted( package.get('name') for package in self.packages_list().json()) def packages_list(self): """Lists the packages defined in openwhisk.""" return self._get(self.gen.url_package()) def package_create(self, filename, package_name): """Uploads contents of the specified file to the specified package.""" pass # TODO def package_delete(self, package_name): """Deletes the specified package.""" pass # TODO def package_info(self, package_name): """Returns info on the specified package.""" pass # TODO # Rules =================================================================== @property def rules(self): """Returns a sorted list of the names of all rules.""" return self.rules_list().json() def rules_list(self): """Lists the rules defined in openwhisk.""" return self._get(self.gen.url_rule()) def rule_create(self, filename, rule_name): """Uploads contents of the specified file to the specified rule.""" pass # TODO def rule_delete(self, rule_name): """Deletes the specified rule.""" pass # TODO def rule_info(self, rule_name): """Returns info on the specified rule.""" pass # TODO # Triggers ================================================================ @property def triggers(self): """Returns a sorted list of the names of all triggers.""" return self.triggers_list().json() def triggers_list(self): """Lists the triggers defined in openwhisk.""" return self._get(self.gen.url_trigger()) def trigger_create(self, filename, trigger_name): """Uploads contents of the specified file to the specified trigger.""" # Read the file into a string # TODO '''with open(filename) as in_file: code = in_file.read() # print('File ' + filename + ' contents: >>>' + code + '<<<') kind = {'py': 'python'}.get(filename.lower().split('.')[-1], 'nodejs') payload = {'exec': {'kind': kind, 'code': code}} return self._put(self.url_actions + action_name, payload)''' def trigger_delete(self, trigger_name): """Deletes the specified package.""" pass # TODO return self._delete(self.url_actions + action_name) def trigger_info(self, trigger_name): """Deletes the specified package.""" pass # TODO return self._delete(self.url_actions + action_name) # Debugging: ============================================================== # These methods will be removed from the final API ====================== # get_a_url('https://openwhisk.ng.bluemix.net/api/v1/namespaces/_/actions/x') def get_a_url(self, url, payload=None): x = self._get(url, payload).json() pprint.pprint(x) return x # post_a_url('https://openwhisk.ng.bluemix.net/api/v1/namespaces/_/actions/x') def post_a_url(self, url, payload=None): return self._post(url, payload).json() @classmethod def _print_request(cls, req_type, url, payload=None): if not DEBUG: return msg = 'Issuing {} request: url={}'.format(req_type, url) if payload: msg += ' payload={!r}'.format(payload) print(msg) @classmethod def _print_response(cls, response): if not DEBUG: return response print('Response.status_code={}\n{}'.format(response.status_code, response.text)) return response def _delete(self, url, payload=None): self._print_request('delete', url, payload) return self._print_response(self.session.delete(url, json=payload)) def _get(self, url, payload=None): self._print_request('get', url, payload) return self._print_response(self.session.get(url, json=payload)) def _post(self, url, payload=None): self._print_request('post', url, payload) return self._print_response(self.session.post(url, json=payload)) def _put(self, url, payload=None): self._print_request('put', url, payload) return self._print_response(self.session.put(url, json=payload)) # Misc utils ============================================================== def invoke_echo(self, message): """Issues a very basic echo request""" echo = '/echo?blocking=true&result=true' return self._post(self.gen.url_whisk_utils + echo, payload={ 'message': message }).json() def system_utils_invoke(self, action_name, **kwargs): """Invokes any action in whisk.system/utils""" url = self.gen.url_whisk_utils + '/' + action_name return self._post(url + '?blocking=true&result=true&', kwargs).json()
def __init__(self, search=''): super().__init__(source='stackoverflow') self.url = UrlGenerator.get_url('stackoverflow', search)
def __init__(self, search=''): super().__init__(source='hh') self.url = UrlGenerator.get_url('hh', search)
def url_generator(): return UrlGenerator(Path(__file__).parent / 'test.json')
def test_missing_configuration(): try: UrlGenerator('non/existing/file') assert False except UrlGeneratorException: assert True
def add_to_queue(self, url): url_gen = UrlGenerator(url, self.word_generator) for url in url_gen.generate_dir_names(): self.queue.put(UrlRequest(url)) for url in url_gen.generate_file_names(): self.queue.put(UrlRequest(url))