def ConfigureHostnames(config, external_hostname = None): """This configures the hostnames stored in the config.""" if not external_hostname: try: external_hostname = socket.gethostname() except (OSError, IOError): print("Sorry, we couldn't guess your hostname.\n") external_hostname = RetryQuestion( "Please enter your hostname e.g. " "grr.example.com", "^[\\.A-Za-z0-9-]+$", external_hostname) print("""\n\n-=Server URL=- The Server URL specifies the URL that the clients will connect to communicate with the server. For best results this should be publicly accessible. By default this will be port 8080 with the URL ending in /control. """) frontend_url = RetryQuestion("Frontend URL", "^http://.*/$", "http://%s:8080/" % external_hostname) config.Set("Client.server_urls", [frontend_url]) frontend_port = urlparse.urlparse(frontend_url).port or grr_config.CONFIG.Get( "Frontend.bind_port") config.Set("Frontend.bind_port", frontend_port) print("""\n\n-=AdminUI URL=-: The UI URL specifies where the Administrative Web Interface can be found. """) ui_url = RetryQuestion("AdminUI URL", "^http[s]*://.*$", "http://%s:8000" % external_hostname) config.Set("AdminUI.url", ui_url) ui_port = urlparse.urlparse(ui_url).port or grr_config.CONFIG.Get( "AdminUI.port") config.Set("AdminUI.port", ui_port)
def __init__(self, uri, transport=None, encoding=None, verbose=False, allow_none=False, use_datetime=False): parsed_url = urlparse(uri) self.__host = uri if parsed_url.scheme else None self.__handler = urlparse(uri).path if not self.__handler: self.__handler = '/' if not transport: transport = SCGITransport(use_datetime=use_datetime) self.__transport = transport self.__encoding = encoding or 'utf-8' self.__verbose = verbose self.__allow_none = allow_none
def get_udp_seeds(url, info_hash): parsed_url = urlparse(url) try: port = parsed_url.port except ValueError as ve: log.error('UDP Port Error, url was %s' % url) return 0 log.debug('Checking for seeds from %s' % url) connection_id = 0x41727101980 # connection id is always this transaction_id = randrange(1, 65535) # Random Transaction ID creation if port is None: log.error('UDP Port Error, port was None') return 0 if port < 0 or port > 65535: log.error('UDP Port Error, port was %s' % port) return 0 # Create the socket try: clisocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) clisocket.settimeout(5.0) clisocket.connect((parsed_url.hostname, port)) # build packet with connection_ID, using 0 value for action, giving our transaction ID for this packet packet = struct.pack(b">QLL", connection_id, 0, transaction_id) clisocket.send(packet) # set 16 bytes ["QLL" = 16 bytes] for the fmq for unpack res = clisocket.recv(16) # check recieved packet for response action, transaction_id, connection_id = struct.unpack(b">LLQ", res) # build packet hash out of decoded info_hash packet_hash = binascii.unhexlify(info_hash) # construct packet for scrape with decoded info_hash setting action byte to 2 for scape packet = struct.pack(b">QLL", connection_id, 2, transaction_id) + packet_hash clisocket.send(packet) # set recieve size of 8 + 12 bytes res = clisocket.recv(20) except IOError as e: log.warning('Socket Error: %s', e) return 0 # Check for UDP error packet (action,) = struct.unpack(b">L", res[:4]) if action == 3: log.error('There was a UDP Packet Error 3') return 0 # first 8 bytes are followed by seeders, completed and leechers for requested torrent seeders, completed, leechers = struct.unpack(b">LLL", res[8:20]) log.debug('get_udp_seeds is returning: %s', seeders) clisocket.close() return seeders
def test_verification_key_is_valid_with_redirect_url_set(self): data = _profile_data() self._create_user_using_profiles_endpoint(data) view = UserProfileViewSet.as_view({'get': 'verify_email'}) rp = RegistrationProfile.objects.get( user__username=data.get('username') ) _data = { 'verification_key': rp.activation_key, 'redirect_url': 'http://red.ir.ect' } request = self.factory.get('/', data=_data) response = view(request) self.assertEquals(response.status_code, 302) self.assertIn('is_email_verified', response.url) self.assertIn('username', response.url) string_query_params = urlparse(response.url).query dict_query_params = parse_qs(string_query_params) self.assertEquals(dict_query_params.get( 'is_email_verified'), ['True']) self.assertEquals( dict_query_params.get('username'), [data.get('username')] ) up = UserProfile.objects.get(user__username=data.get('username')) self.assertIn('is_email_verified', up.metadata) self.assertTrue(up.metadata.get('is_email_verified'))
def single_request(self, host, handler, request_body, verbose=0): # Add SCGI headers to the request. headers = [('CONTENT_LENGTH', native_str(len(request_body))), ('SCGI', '1')] header = '\x00'.join(['%s\x00%s' % (key, value) for key, value in headers]) + '\x00' header = '%d:%s' % (len(header), header) request_body = '%s,%s' % (header, request_body) sock = None try: if host: parsed_host = urlparse(host) host = parsed_host.hostname port = parsed_host.port addr_info = socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM) sock = socket.socket(*addr_info[0][:3]) sock.connect(addr_info[0][4]) else: sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.connect(handler) self.verbose = verbose sock.sendall(request_body.encode()) return self.parse_response(sock.makefile()) finally: if sock: sock.close()
def testEmailCronJobApprovalRequestLinkLeadsToACorrectPage(self): job_name = self._CreateOSBreakDownCronJobApproval() self.RequestCronJobApproval( job_name, reason=self.APPROVAL_REASON, approver=self.GRANTOR_USERNAME, requestor=self.token.username) self.assertEqual(len(self.messages_sent), 1) message = self.messages_sent[0] self.assertIn(self.APPROVAL_REASON, message) self.assertIn(self.token.username, message) self.assertIn("OSBreakDown", message) # Extract link from the message text and open it. m = re.search(r"href='(.+?)'", message, re.MULTILINE) link = urlparse.urlparse(m.group(1)) self.Open(link.path + "?" + link.query + "#" + link.fragment) # Check that requestor's username and reason are correctly displayed. self.WaitUntil(self.IsTextPresent, self.token.username) self.WaitUntil(self.IsTextPresent, self.APPROVAL_REASON) # Check that host information is displayed. self.WaitUntil(self.IsTextPresent, cron_system.OSBreakDown.__name__) self.WaitUntil(self.IsTextPresent, "Frequency")
def url_rewrite(self, task, entry): log.debug('Requesting %s' % entry['url']) page = requests.get(entry['url']) soup = get_soup(page.text) for link in soup.findAll('a', attrs={'href': re.compile(r'^/url')}): # Extract correct url from google internal link href = 'http://google.com' + link['href'] args = parse_qs(urlparse(href).query) href = args['q'][0] # import IPython; IPython.embed() # import sys # sys.exit(1) # href = link['href'].lstrip('/url?q=').split('&')[0] # Test if entry with this url would be recognized by some urlrewriter log.trace('Checking if %s is known by some rewriter' % href) fake_entry = {'title': entry['title'], 'url': href} urlrewriting = plugin.get_plugin_by_name('urlrewriting') if urlrewriting['instance'].url_rewritable(task, fake_entry): log.debug('--> rewriting %s (known url pattern)' % href) entry['url'] = href return else: log.debug('<-- ignoring %s (unknown url pattern)' % href) raise UrlRewritingError('Unable to resolve')
def url_rewrite(self, task, entry): url = entry['url'] page = None for (scheme, netloc) in EZTV_MIRRORS: try: _, _, path, params, query, fragment = urlparse(url) url = urlunparse((scheme, netloc, path, params, query, fragment)) page = task.requests.get(url).content except RequestException as e: log.debug('Eztv mirror `%s` seems to be down', url) continue break if not page: raise UrlRewritingError('No mirrors found for url %s' % entry['url']) log.debug('Eztv mirror `%s` chosen', url) try: soup = get_soup(page) mirrors = soup.find_all('a', attrs={'class': re.compile(r'download_\d')}) except Exception as e: raise UrlRewritingError(e) log.debug('%d torrent mirrors found', len(mirrors)) if not mirrors: raise UrlRewritingError('Unable to locate download link from url %s' % url) entry['urls'] = [m.get('href') for m in mirrors] entry['url'] = mirrors[0].get('href')
def _parse_urls(self, urls): parsed_urls = [urlparse(url) for url in urls] for parsed_url in parsed_urls: if not parsed_url.scheme in self.SUPPORTED_PROTOCOLS: raise InvalidRESTfulCacheProtoError( 'RESTfulArtifactCache only supports HTTP(S). Found: {0}'.format(parsed_url.scheme)) return parsed_urls
def program_id_from_url(self, url): parsed = urlparse(url) query_dict = parse_qs(parsed.query) if query_dict.get('_c'): return query_dict.get('_c')[0] else: return parsed.path.split('/')[-1]
def _domains_base_url(url): url_parts = urlparse(url) return "{scheme}://{domain}/".format( scheme=url_parts.scheme, domain=url_parts.netloc )
def process_invalid_content(self, task, data, url): """If feedparser reports error, save the received data and log error.""" if data is None: log.critical('Received empty page - no content') return else: data = tobytes(data) ext = 'xml' if b'<html>' in data.lower(): log.critical('Received content is HTML page, not an RSS feed') ext = 'html' if b'login' in data.lower() or b'username' in data.lower(): log.critical('Received content looks a bit like login page') if b'error' in data.lower(): log.critical('Received content looks a bit like error page') received = os.path.join(task.manager.config_base, 'received') if not os.path.isdir(received): os.mkdir(received) filename = task.name sourcename = urlparse(url).netloc if sourcename: filename += '-' + sourcename filename = pathscrub(filename, filename=True) filepath = os.path.join(received, '%s.%s' % (filename, ext)) with open(filepath, 'wb') as f: f.write(data) log.critical('I have saved the invalid content to %s for you to view', filepath)
def test_enter_data_redir(self): if not self._running_enketo(): raise SkipTest with HTTMock(enketo_mock): factory = RequestFactory() request = factory.get("/") request.user = self.user response = enter_data( request, self.user.username, self.xform.id_string ) # make sure response redirect to an enketo site enketo_base_url = urlparse(settings.ENKETO_URL).netloc redirected_base_url = urlparse(response["Location"]).netloc # TODO: checking if the form is valid on enketo side self.assertIn(enketo_base_url, redirected_base_url) self.assertEqual(response.status_code, 302)
def get_hosts(urls): """Get hosts extracted from URLs. :param urls: URL addresses from which the function extracts hosts :returns: a list of hosts extracted from the URLs """ return [urlparse(u).hostname for u in urls]
def show_terms_if_not_agreed(context, field=TERMS_HTTP_PATH_FIELD): """Displays a modal on a current page if a user has not yet agreed to the given terms. If terms are not specified, the default slug is used. How it works? A small snippet is included into your template if a user who requested the view has not yet agreed the terms. The snippet takes care of displaying a respective modal. """ request = context["request"] all_agreed = True not_agreed_terms = [] for terms in TermsAndConditions.get_active_list(as_dict=False): if not TermsAndConditions.agreed_to_terms(request.user, terms): all_agreed = False not_agreed_terms.append(terms) # stop here, if all terms have been agreed if all_agreed: return {} # handle excluded url's url = urlparse(request.META[field]) protected = is_path_protected(url.path) if (not all_agreed) and not_agreed_terms and protected: return {"not_agreed_terms": not_agreed_terms, "returnTo": url.path} return {}
def get_owner_and_repo(repourl): """ Takes a git repository URL from Bitbucket and tries to determine the owner and repository name :param repourl: Bitbucket git repo in the form of [email protected]:OWNER/REPONAME.git https://bitbucket.com/OWNER/REPONAME.git ssh://[email protected]/OWNER/REPONAME.git :return: owner, repo: The owner of the repository and the repository name """ parsed = urlparse(repourl) if parsed.scheme: path = parsed.path[1:] else: # we assume git@host:owner/repo.git here path = parsed.path.split(':', 1)[-1] if path.endswith('.git'): path = path[:-4] while path.endswith('/'): path = path[:-1] parts = path.split('/') assert len(parts) == 2, 'OWNER/REPONAME is expected' return parts
def _predicted_urls(url): domain = urlparse(url).netloc return { 'http://{}/rss/'.format(domain), 'http://{}/feeds/'.format(domain), 'http://{}/feed/'.format(domain) }
def get_password_reset_email(user, reset_url, subject_template_name='registration/password_reset_subject.txt', # noqa email_template_name='api_password_reset_email.html', # noqa token_generator=default_token_generator, email_subject=None): """Creates the subject and email body for password reset email.""" result = urlparse(reset_url) site_name = domain = result.hostname encoded_username = urlsafe_base64_encode( b(user.username.encode('utf-8'))) c = { 'email': user.email, 'domain': domain, 'path': result.path, 'site_name': site_name, 'uid': urlsafe_base64_encode(force_bytes(user.pk)), 'username': user.username, 'encoded_username': encoded_username, 'token': token_generator.make_token(user), 'protocol': result.scheme if result.scheme != '' else 'http', } # if subject email provided don't load the subject template subject = email_subject or loader.render_to_string(subject_template_name, c) # Email subject *must not* contain newlines subject = ''.join(subject.splitlines()) email = loader.render_to_string(email_template_name, c) return subject, email
def to_internal_value(self, data): try: http_prefix = data.startswith(('http:', 'https:')) except AttributeError: self.fail('incorrect_type', data_type=type(data).__name__) input_data = data if http_prefix: # If needed convert absolute URLs to relative path data = urlparse(data).path prefix = get_script_prefix() if data.startswith(prefix): data = '/' + data[len(prefix):] try: match = self.resolve(data) except Resolver404: self.fail('no_match') if match.view_name not in self.view_names: self.fail('incorrect_match', input=input_data) self._setup_field(match.view_name) try: return self.get_object(match.view_name, match.args, match.kwargs) except (ObjectDoesNotExist, TypeError, ValueError): self.fail('does_not_exist') return data
def url_rewrite(self, task, entry): url = entry["url"] page = None for (scheme, netloc) in EZTV_MIRRORS: try: _, _, path, params, query, fragment = urlparse(url) url = urlunparse((scheme, netloc, path, params, query, fragment)) page = task.requests.get(url).content except RequestException as e: log.debug("Eztv mirror `%s` seems to be down", url) continue break if not page: raise UrlRewritingError("No mirrors found for url %s" % entry["url"]) log.debug("Eztv mirror `%s` chosen", url) try: soup = get_soup(page) mirrors = soup.find_all("a", attrs={"class": re.compile(r"download_\d")}) except Exception as e: raise UrlRewritingError(e) log.debug("%d torrent mirrors found", len(mirrors)) if not mirrors: raise UrlRewritingError("Unable to locate download link from url %s" % url) entry["urls"] = [m.get("href") for m in mirrors] entry["url"] = mirrors[0].get("href")
def on_task_download(self, task, config): config = self.prepare_config(config, task) for entry in task.accepted: ftp_url = urlparse(entry.get('url')) ftp_url = ftp_url._replace(path=unquote(ftp_url.path)) current_path = os.path.dirname(ftp_url.path) try: ftp = self.ftp_connect(config, ftp_url, current_path) except ftplib.all_errors as e: entry.fail("Unable to connect to server : %s" % (e)) break if not os.path.isdir(config['ftp_tmp_path']): log.debug('creating base path: %s' % config['ftp_tmp_path']) os.mkdir(config['ftp_tmp_path']) file_name = os.path.basename(ftp_url.path) try: # Directory ftp = self.check_connection(ftp, config, ftp_url, current_path) ftp.cwd(file_name) self.ftp_walk(ftp, os.path.join(config['ftp_tmp_path'], file_name), config, ftp_url, ftp_url.path) ftp = self.check_connection(ftp, config, ftp_url, current_path) ftp.cwd('..') if config['delete_origin']: ftp.rmd(file_name) except ftplib.error_perm: # File self.ftp_down(ftp, file_name, config['ftp_tmp_path'], config, ftp_url, current_path) ftp.close()
def GetCurrentUrlPath(self): url = urlparse.urlparse(self.driver.current_url) result = url.path if url.fragment: result += "#" + url.fragment return result
def set_urls(self, url): url = url.rstrip('/') if self.url != url: self.url = url parsed_url = urlparse(url) self.url_match = re.compile('^%s://(?:torrents\.)?(%s)/.*$' % (re.escape(parsed_url.scheme), re.escape(parsed_url.netloc))) self.url_search = re.compile('^%s/search/.*$' % (re.escape(url)))
def program_id_from_url(self, url): parsed = urlparse(url) query_dict = parse_qs(parsed.query) play = query_dict.get('play') if parsed.path.startswith('/tv/ohjelmat/') and play: return play[0] else: return parsed.path.split('/')[-1]
def is_valid_url(value): """Check if given value is a valid URL string. :param value: a value to test :returns: True if the value is valid """ match = URL_REGEX.match(value) host_str = urlparse(value).hostname return match and is_valid_host(host_str)
def call_url(self, expected_url, with_error=False): try: with self.best_url_selector.select_best_url() as url: self.assertEquals(urlparse(expected_url), url) if with_error: raise RequestException('error connecting to {}'.format(url)) except RequestException: pass
def on_task_exit(self, task, config): config = self.prepare_config(config) if not config['enabled'] or task.options.learn: return if not self.client: self.client = self.create_rpc_client(config) tracker_re = re.compile(config['tracker'], re.IGNORECASE) if 'tracker' in config else None preserve_tracker_re = ( re.compile(config['preserve_tracker'], re.IGNORECASE) if 'preserve_tracker' in config else None ) session = self.client.get_session() remove_ids = [] for torrent in self.client.get_torrents(): log.verbose( 'Torrent "%s": status: "%s" - ratio: %s - date added: %s' % (torrent.name, torrent.status, torrent.ratio, torrent.date_added) ) downloaded, dummy = self.torrent_info(torrent, config) if not downloaded: continue if config.get('transmission_seed_limits'): seed_ratio_ok, idle_limit_ok = self.check_seed_limits(torrent, session) if not seed_ratio_ok or not idle_limit_ok: continue if 'min_ratio' in config: if torrent.ratio < config['min_ratio']: continue if 'finished_for' in config: # done date might be invalid if this torrent was added to transmission when already completed started_seeding = datetime.fromtimestamp(max(torrent.addedDate, torrent.doneDate)) if started_seeding + parse_timedelta(config['finished_for']) > datetime.now(): continue tracker_hosts = ( urlparse(tracker['announce']).hostname for tracker in torrent.trackers ) if 'tracker' in config: if not any(tracker_re.search(tracker) for tracker in tracker_hosts): continue if 'preserve_tracker' in config: if any(preserve_tracker_re.search(tracker) for tracker in tracker_hosts): continue if config.get('directories'): if not any( re.search(d, torrent.downloadDir, re.IGNORECASE) for d in config['directories'] ): continue if task.options.test: log.info('Would remove finished torrent `%s` from transmission', torrent.name) continue log.info('Removing finished torrent `%s` from transmission', torrent.name) remove_ids.append(torrent.id) if remove_ids: self.client.remove_torrent(remove_ids, config.get('delete_files'))
def any_match(self, urls): """Check if any of the given URLs has a matching host. :param urls: an iterable containing URLs :returns: True if any host has a listed match :raises InvalidURLError: if there are any invalid URLs in the sequence """ return any(urlparse(u).hostname in self for u in urls)
def redirect_to_terms_accept(current_path='/', slug='default'): """Redirect the user to the terms and conditions accept page.""" redirect_url_parts = list(urlparse(ACCEPT_TERMS_PATH)) if slug != 'default': redirect_url_parts[2] += slug querystring = QueryDict(redirect_url_parts[4], mutable=True) querystring[TERMS_RETURNTO_PARAM] = current_path redirect_url_parts[4] = querystring.urlencode(safe='/') return HttpResponseRedirect(urlunparse(redirect_url_parts))
def is_unresponsive(url): """ Checks if host of given url has timed out within WAIT_TIME :param url: The url to check :return: True if the host has timed out within WAIT_TIME :rtype: bool """ host = urlparse(url).hostname return host in unresponsive_hosts
def is_safe_url(target): ref_url = urlparse(request.host_url) test_url = urlparse(urljoin(request.host_url, target)) return test_url.scheme in ('http', 'https') and ref_url.netloc == test_url.netloc
def on_task_exit(self, task, config): config = self.prepare_config(config) if not config['enabled'] or task.options.learn: return if not self.client: self.client = self.create_rpc_client(config) nrat = float(config['min_ratio']) if 'min_ratio' in config else None nfor = parse_timedelta( config['finished_for']) if 'finished_for' in config else None delete_files = bool( config['delete_files']) if 'delete_files' in config else False trans_checks = bool( config['transmission_seed_limits'] ) if 'transmission_seed_limits' in config else False tracker_re = re.compile(config['tracker'], re.IGNORECASE) if 'tracker' in config else None directories_re = config.get('directories') session = self.client.get_session() remove_ids = [] for torrent in self.client.get_torrents(): log.verbose( 'Torrent "%s": status: "%s" - ratio: %s - date added: %s - date done: %s' % (torrent.name, torrent.status, torrent.ratio, torrent.date_added, torrent.date_done)) downloaded, dummy = self.torrent_info(torrent, config) seed_ratio_ok, idle_limit_ok = self.check_seed_limits( torrent, session) tracker_hosts = (urlparse(tracker['announce']).hostname for tracker in torrent.trackers) is_clean_all = nrat is None and nfor is None and trans_checks is None is_minratio_reached = nrat and (nrat <= torrent.ratio) is_transmission_seedlimit_unset = trans_checks and seed_ratio_ok is None and idle_limit_ok is None is_transmission_seedlimit_reached = trans_checks and seed_ratio_ok is True is_transmission_idlelimit_reached = trans_checks and idle_limit_ok is True is_torrent_seed_only = torrent.date_done <= torrent.date_added is_torrent_idlelimit_since_added_reached = nfor and ( torrent.date_added + nfor) <= datetime.now() is_torrent_idlelimit_since_finished_reached = nfor and ( torrent.date_done + nfor) <= datetime.now() is_tracker_matching = not tracker_re or any( tracker_re.search(host) for host in tracker_hosts) is_directories_matching = not directories_re or any( re.compile(directory, re.IGNORECASE).search( torrent.downloadDir) for directory in directories_re) if (downloaded and (is_clean_all or is_transmission_seedlimit_unset or is_transmission_seedlimit_reached or is_transmission_idlelimit_reached or is_minratio_reached or (is_torrent_seed_only and is_torrent_idlelimit_since_added_reached) or (not is_torrent_seed_only and is_torrent_idlelimit_since_finished_reached)) and is_tracker_matching and is_directories_matching): if task.options.test: log.info( 'Would remove finished torrent `%s` from transmission' % torrent.name) continue log.info('Removing finished torrent `%s` from transmission' % torrent.name) remove_ids.append(torrent.id) if remove_ids: self.client.remove_torrent(remove_ids, delete_files)
def program_id_from_url(self, url): parsed = urlparse(url) return parsed.path.split('/')[-1]
def download_flavors(self, download_url, media_type): path = urlparse(download_url)[2] ext = os.path.splitext(path)[1] or None backend = WgetBackend(download_url, ext) return [StreamFlavor(media_type=media_type, streams=[backend])]
def get_udp_seeds(url, info_hash): parsed_url = urlparse(url) try: port = parsed_url.port except ValueError: log.error('UDP Port Error, url was %s', url) return 0 log.debug('Checking for seeds from %s', url) connection_id = 0x41727101980 # connection id is always this transaction_id = randrange(1, 65535) # Random Transaction ID creation if port is None: log.error('UDP Port Error, port was None') return 0 if port < 0 or port > 65535: log.error('UDP Port Error, port was %s', port) return 0 # Create the socket try: clisocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) clisocket.settimeout(5.0) clisocket.connect((parsed_url.hostname, port)) # build packet with connection_ID, using 0 value for action, giving our transaction ID for this packet packet = struct.pack(b">QLL", connection_id, 0, transaction_id) clisocket.send(packet) # set 16 bytes ["QLL" = 16 bytes] for the fmq for unpack res = clisocket.recv(16) # check recieved packet for response action, transaction_id, connection_id = struct.unpack(b">LLQ", res) # build packet hash out of decoded info_hash packet_hash = binascii.unhexlify(info_hash) # construct packet for scrape with decoded info_hash setting action byte to 2 for scape packet = struct.pack(b">QLL", connection_id, 2, transaction_id) + packet_hash clisocket.send(packet) # set recieve size of 8 + 12 bytes res = clisocket.recv(20) except IOError as e: log.warning('Socket Error: %s', e) return 0 # Check for UDP error packet (action, ) = struct.unpack(b">L", res[:4]) if action == 3: log.error('There was a UDP Packet Error 3') return 0 # first 8 bytes are followed by seeders, completed and leechers for requested torrent seeders, _, _ = struct.unpack(b">LLL", res[8:20]) log.debug('get_udp_seeds is returning: %s', seeders) clisocket.close() return seeders
def parse_url(url, proto='http'): # urlparse won't parse properly without a protocol if '://' not in url: url = proto + '://' + url return urlparse(url)
def __init__(self, url): parsed_url = urlparse(url) self.url = url self.server = parsed_url.hostname.lower() self.has_ssl = parsed_url.scheme == 'https' super(RedirectError, self).__init__(text_type(self))
def _create_beam_sdk(sdk_remote_location, temp_dir): # type: (...) -> List[beam_runner_api_pb2.ArtifactInformation] """Creates a Beam SDK file with the appropriate version. Args: sdk_remote_location: A URL from which the file can be downloaded or a remote file location. The SDK file can be a tarball or a wheel. Set to 'pypi' to download and stage a wheel and source SDK from PyPi. temp_dir: path to temporary location where the file should be downloaded. Returns: A list of ArtifactInformation of local files path and SDK files that will be staged to the staging location. Raises: RuntimeError: if staging was not successful. """ if sdk_remote_location == 'pypi': sdk_local_file = Stager._download_pypi_sdk_package(temp_dir) sdk_sources_staged_name = Stager.\ _desired_sdk_filename_in_staging_location(sdk_local_file) _LOGGER.info('Staging SDK sources from PyPI: %s', sdk_sources_staged_name) staged_sdk_files = [ Stager._create_file_stage_to_artifact(sdk_local_file, sdk_sources_staged_name) ] try: abi_suffix = 'm' if sys.version_info < (3, 8) else '' # Stage binary distribution of the SDK, for now on a best-effort basis. sdk_local_file = Stager._download_pypi_sdk_package( temp_dir, fetch_binary=True, language_version_tag='%d%d' % (sys.version_info[0], sys.version_info[1]), abi_tag='cp%d%d%s' % (sys.version_info[0], sys.version_info[1], abi_suffix)) sdk_binary_staged_name = Stager.\ _desired_sdk_filename_in_staging_location(sdk_local_file) _LOGGER.info( 'Staging binary distribution of the SDK from PyPI: %s', sdk_binary_staged_name) staged_sdk_files.append( Stager._create_file_stage_to_artifact( sdk_local_file, sdk_binary_staged_name)) except RuntimeError as e: _LOGGER.warning( 'Failed to download requested binary distribution ' 'of the SDK: %s', repr(e)) return staged_sdk_files elif Stager._is_remote_path(sdk_remote_location): sdk_remote_parsed = urlparse(sdk_remote_location) sdk_remote_filename = os.path.basename(sdk_remote_parsed.path) local_download_file = os.path.join(temp_dir, sdk_remote_filename) Stager._download_file(sdk_remote_location, local_download_file) staged_name = Stager._desired_sdk_filename_in_staging_location( local_download_file) _LOGGER.info('Staging Beam SDK from %s', sdk_remote_location) return [ Stager._create_file_stage_to_artifact(local_download_file, staged_name) ] else: raise RuntimeError( 'The --sdk_location option was used with an unsupported ' 'type of location: %s' % sdk_remote_location)
def publish(self, user, id_string=None, created_by=None): """ Publish XLSForm. """ if self.is_valid(): # If a text (csv) representation of the xlsform is present, # this will save the file and pass it instead of the 'xls_file' # field. cleaned_xls_file = None if 'text_xls_form' in self.cleaned_data\ and self.cleaned_data['text_xls_form'].strip(): csv_data = self.cleaned_data['text_xls_form'] # assigning the filename to a random string (quick fix) import random rand_name = "uploaded_form_%s.csv" % ''.join( random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6)) cleaned_xls_file = \ default_storage.save( upload_to(None, rand_name, user.username), ContentFile(csv_data)) if 'xls_file' in self.cleaned_data and\ self.cleaned_data['xls_file']: cleaned_xls_file = self.cleaned_data['xls_file'] if 'floip_file' in self.cleaned_data and\ self.cleaned_data['floip_file']: cleaned_xls_file = self.cleaned_data['floip_file'] cleaned_url = ( self.cleaned_data['xls_url'].strip() or self.cleaned_data['dropbox_xls_url'] or self.cleaned_data['csv_url']) if cleaned_url: cleaned_xls_file = urlparse(cleaned_url) cleaned_xls_file = \ '_'.join(cleaned_xls_file.path.split('/')[-2:]) name, extension = os.path.splitext(cleaned_xls_file) if extension not in VALID_FILE_EXTENSIONS and name: response = requests.get(cleaned_url) if response.headers.get('content-type') in \ VALID_XLSFORM_CONTENT_TYPES and \ response.status_code < 400: cleaned_xls_file = get_filename(response) cleaned_xls_file = \ upload_to(None, cleaned_xls_file, user.username) self.validate(cleaned_url) xls_data = ContentFile(urlopen(cleaned_url).read()) cleaned_xls_file = \ default_storage.save(cleaned_xls_file, xls_data) project = self.cleaned_data['project'] if project is None: project = get_user_default_project(user) else: project = self._project cleaned_xml_file = self.cleaned_data['xml_file'] if cleaned_xml_file: return publish_xml_form(cleaned_xml_file, user, project, id_string, created_by or user) if cleaned_xls_file is None: raise forms.ValidationError( _(u"XLSForm not provided, expecting either of these" " params: 'xml_file', 'xls_file', 'xls_url', 'csv_url'," " 'dropbox_xls_url', 'text_xls_form', 'floip_file'")) # publish the xls return publish_xls_form(cleaned_xls_file, user, project, id_string, created_by or user)
def movie_delete_request(base_url, port, api_key): parsedurl = urlparse(base_url) log.debug('Received movie delete request') return '%s://%s:%s%s/api/%s/movie.delete?delete_from=wanted' % ( parsedurl.scheme, parsedurl.netloc, port, parsedurl.path, api_key)
def movie_add_request(base_url, port, api_key): parsedurl = urlparse(base_url) log.debug('Received movie add request') return '%s://%s:%s%s/api/%s/movie.add' % ( parsedurl.scheme, parsedurl.netloc, port, parsedurl.path, api_key)
def file_extension(self): """The file extension of :attr:`~.Media.url`. Read-only. :type: :obj:`str` """ return '.' + urlparse(self.url).path.split('.')[-1]
def profile_list_request(base_url, port, api_key): parsedurl = urlparse(base_url) log.debug('Received profile list request') return '%s://%s:%s%s/api/%s/profile.list' % ( parsedurl.scheme, parsedurl.netloc, port, parsedurl.path, api_key)
def install_frontend(self, source, oauth_key, oauth_secret, backend_url, settings_file=None, network='public', hub_id=None): """ Install connector-frontend in Odin Automation Hub, --source can be http(s):// or filepath""" with TemporaryDirectory() as tdir: is_http_source = True if source.startswith('http://') or source.startswith('https://') \ else False if is_http_source: package_name = _download_file(source, target=tdir) else: package_name = os.path.basename(source) copyfile(os.path.expanduser(source), os.path.join(tdir, package_name)) package_path = os.path.join(tdir, package_name) with zipfile.ZipFile(package_path, 'r') as zip_ref: meta_path = zip_ref.extract('APP-META.xml', path=tdir) tenant_schema_path = zip_ref.extract('schemas/tenant.schema', tdir) app_schema_path = zip_ref.extract('schemas/app.schema', tdir) try: zip_ref.extract('schemas/user.schema', tdir) user_service = True except KeyError: user_service = False tree = xml_et.ElementTree(file=meta_path) namespace = '{http://aps-standard.org/ns/2}' connector_id = tree.find('{}id'.format(namespace)).text version = tree.find('{}version'.format(namespace)).text release = tree.find('{}release'.format(namespace)).text # Get connector name from id as <name> field may not be unique url_path = urlparse(connector_id).path connector_name = os.path.split(url_path)[-1] if not settings_file: settings_file = {} else: settings_file = json.load(open(settings_file)) if backend_url.startswith('http://'): print( "WARN: Make sure that the APS development mode enabled for http backend. " "Run `apsconnect aps_devel_mode` command.") elif backend_url.startswith('https://'): pass else: print("Backend url must be URL http(s)://, got {}".format( backend_url)) sys.exit(1) cfg, hub = _get_cfg(), _get_hub() with open(package_path, 'rb') as package_binary: print("Importing connector {} {}-{}".format( connector_id, version, release)) import_kwargs = {'package_url': source} if is_http_source \ else {'package_body': xmlrpclib.Binary(package_binary.read())} response = hub.APS.importPackage(**import_kwargs) _osaapi_raise_for_status(response) application_id = str(response['result']['application_id']) print("Connector {} imported with id={} [ok]".format( connector_id, application_id)) payload = { 'aps': { 'package': { 'type': connector_id, 'version': version, 'release': release, }, 'endpoint': backend_url, 'network': network, 'auth': { 'oauth': { 'key': oauth_key, 'secret': oauth_secret, }, }, }, } # Get Unique OA id for using as hubId parameter while endpoint deploying base_aps_url = _get_aps_url( **{k: _get_cfg()[k] for k in APS_CONNECT_PARAMS}) app_properties = _get_properties(app_schema_path) if 'hubId' in app_properties: url = '{}/{}'.format( base_aps_url, 'aps/2/resources?implementing(http://parallels.com/aps/types/pa/poa/1.0)', ) response = request(method='GET', url=url, headers=_get_user_token(hub, cfg['user']), verify=False) response.raise_for_status() try: data = json.loads(response.content.decode('utf-8')) except ValueError: print("APSController provided non-json format") sys.exit(1) if not data and not hub_id: raise Exception( "Core OA resource is not found\n" "Use --hub-id={value} argument to specify the ID " "manually or --hub-id=auto to generate it automatically" ) elif data: hub_id = data[0]['aps']['id'] elif hub_id == 'auto': hub_id = str(uuid.uuid4()) payload.update({'app': {'hubId': hub_id}}) payload.update(settings_file) response = request(method='POST', url='{}/{}'.format(base_aps_url, 'aps/2/applications/'), headers=_get_user_token(hub, cfg['user']), verify=False, json=payload) try: response.raise_for_status() except Exception as e: if 'error' in response.json(): err = "{} {}".format(response.json()['error'], response.json()['message']) else: err = str(e) print("Installation of connector {} FAILED.\n" "Hub APS API response {} code.\n" "Error: {}".format(connector_id, response.status_code, err)) # Create app, tenant, users resource types resource_uid = json.loads( response.content.decode('utf-8'))['app']['aps']['id'] core_resource_types_payload = [ { 'resclass_name': 'rc.saas.service.link', 'name': connector_name, 'act_params': [ { 'var_name': 'app_id', 'var_value': application_id }, { 'var_name': 'resource_uid', 'var_value': resource_uid }, ] }, { 'resclass_name': 'rc.saas.service', 'name': '{} tenant'.format(connector_name), 'act_params': [ { 'var_name': 'app_id', 'var_value': application_id }, { 'var_name': 'service_id', 'var_value': 'tenant' }, { 'var_name': 'autoprovide_service', 'var_value': '1' }, ] }, ] # Collect ids for service template creation resource_types_ids = [] limited_resources = {} for type in core_resource_types_payload: response = hub.addResourceType(**type) _osaapi_raise_for_status(response) resource_types_ids.append( response['result']['resource_type_id']) for id in list(resource_types_ids): limited_resources[id] = 1 if user_service: user_resource_type_payload = { 'resclass_name': 'rc.saas.service', 'name': '{} users'.format(connector_name), 'act_params': [ { 'var_name': 'app_id', 'var_value': application_id }, { 'var_name': 'service_id', 'var_value': 'user' }, { 'var_name': 'autoprovide_service', 'var_value': '0' }, ] } response = hub.addResourceType(**user_resource_type_payload) _osaapi_raise_for_status(response) resource_types_ids.append( response['result']['resource_type_id']) # Create counters resource types counters = _get_counters(tenant_schema_path) for counter in counters: payload = { 'resclass_name': "rc.saas.resource.unit", 'name': '{} {}'.format(connector_name, counter), 'act_params': [ { 'var_name': 'app_id', 'var_value': application_id }, { 'var_name': 'service_id', 'var_value': "tenant" }, { 'var_name': 'resource_id', 'var_value': counter }, ] } response = hub.addResourceType(**payload) _osaapi_raise_for_status(response) resource_types_ids.append( response['result']['resource_type_id']) # Create parameters resource types parameters = _get_parameters(tenant_schema_path) for parameter in parameters: payload = { 'resclass_name': "rc.saas.resource.unit", 'name': '{} {}'.format(connector_name, parameter), 'act_params': [ { 'var_name': 'app_id', 'var_value': application_id }, { 'var_name': 'service_id', 'var_value': "tenant" }, { 'var_name': 'resource_id', 'var_value': parameter }, ] } response = hub.addResourceType(**payload) _osaapi_raise_for_status(response) resource_types_ids.append( response['result']['resource_type_id']) limited_resources[response['result']['resource_type_id']] = 0 print("Resource types creation [ok]") # Create service template payload = { 'name': connector_name, 'owner_id': 1, 'resources': [], } for type_id in resource_types_ids: payload['resources'].append({'resource_type_id': type_id}) response = hub.addServiceTemplate(**payload) _osaapi_raise_for_status(response) service_template_id = response['result']['st_id'] print("Service template \"{}\" created with id={} [ok]".format( connector_name, service_template_id)) # Set up limits payload = { 'st_id': service_template_id, 'limits': [], } for type_id, limit in limited_resources.items(): payload['limits'].append({ 'resource_id': type_id, 'resource_limit64': str(limit) }) response = hub.setSTRTLimits(**payload) _osaapi_raise_for_status(response) print("Limits for Service template \"{}\" are applied [ok]".format( service_template_id))
def _ExtractLinkFromMessage(self, message): m = re.search(r"href='(.+?)'", message, re.MULTILINE) link = urlparse.urlparse(m.group(1)) return link.path + "/" + "#" + link.fragment
def _baseurl(self, url): parsed_url = urlparse(url) return '{scheme}://{netloc}'.format(scheme=parsed_url.scheme, netloc=parsed_url.netloc)
def on_task_input(self, task, config): """ This plugin returns ALL of the shows monitored by Sickbeard. This includes both ongoing and ended. Syntax: sickbeard: base_url=<value> port=<value> api_key=<value> Options base_url and api_key are required. Use with input plugin like discover and/or configure_series. Example: download-tv-task: configure_series: settings: quality: - 720p from: sickbeard: base_url: http://localhost port: 8531 api_key: MYAPIKEY1123 discover: what: - next_series_episodes: yes from: torrentz: any download: /download/tv Note that when using the configure_series plugin with Sickbeard you are basically synced to it, so removing a show in Sickbeard will remove it in flexget as well, which could be positive or negative, depending on your usage. """ parsedurl = urlparse(config.get('base_url')) url = '%s://%s:%s%s/api/%s/?cmd=shows' % ( parsedurl.scheme, parsedurl.netloc, config.get('port'), parsedurl.path, config.get('api_key')) try: json = task.requests.get(url).json() except RequestException as e: raise plugin.PluginError( 'Unable to connect to Sickbeard at %s://%s:%s%s. Error: %s' % (parsedurl.scheme, parsedurl.netloc, config.get('port'), parsedurl.path, e)) entries = [] for _, show in list(json['data'].items()): log.debug('processing show: {}'.format(show)) fg_qualities = '' # Initializes the quality parameter if show['paused'] and config.get('only_monitored'): continue if show['status'] == 'Ended' and not config.get('include_ended'): continue if config.get('include_data'): show_url = '%s:%s/api/%s/?cmd=show&tvdbid=%s' % ( config['base_url'], config['port'], config['api_key'], show['tvdbid']) show_json = task.requests.get(show_url).json() log.deubg('processing show data: {}'.format(show_json['data'])) fg_qualities = self.quality_requirement_builder( show_json['data']['quality_details']['initial']) entry = Entry(title=show['show_name'], url='', series_name=show['show_name'], tvdb_id=show.get('tvdbid'), tvrage_id=show.get('tvrage_id')) if len(fg_qualities) > 1: entry['configure_series_qualities'] = fg_qualities elif len(fg_qualities) == 1: entry['configure_series_quality'] = fg_qualities[0] else: entry['configure_series_quality'] = fg_qualities if entry.isvalid(): entries.append(entry) else: log.error('Invalid entry created? %s' % entry) continue # Test mode logging if task.options.test: log.info("Test mode. Entry includes:") for key, value in list(entry.items()): log.info(' {}: {}'.format(key.capitalize(), value)) return entries
def urlparse_drop_netloc(url): url = urlparse(url) if url[4]: return url[2] + '?' + url[4] return url[2]
def is_tv_ohjelmat_url(self, url): return urlparse(url).path.startswith('/tv/ohjelmat/')
def prepare_request(self, method, uri, params=None, headers=None, data=None, json=None, access_token=None): params = {} if params is None else params if not isinstance(params, dict): raise TypeError('params should be dict') if uri == '': uri = '/' method = method.upper() params = utf8_encoded_dict(params) url = '/'.join([self.api_server, self.endpoint, self.version ]) + uri.strip() logger.debug(url) url_parsed = urlparse(url) enc_params = urlencode(params) logger.debug(enc_params) if url_parsed.query == '' or url_parsed.query is None: query = enc_params elif enc_params == '' or enc_params is None: query = url_parsed.query else: query = '%s&%s' % (url_parsed.query, enc_params) real_uri = urlunparse(('', '', url_parsed.path, url_parsed.params, query, url_parsed.fragment)) real_url = urlunparse( (url_parsed.scheme, url_parsed.netloc, url_parsed.path, url_parsed.params, query, url_parsed.fragment)) self.request_data.uri = real_uri self.request_data.method = method self.request_data.headers = { # 'Accept': 'application/json; charset=utf-8', 'Host': url_parsed.netloc } if headers is not None and isinstance(headers, dict): # headers 是字典 self.request_data.headers.update(headers) if access_token is not None: self.request_data.headers[HEADER_X_ACCESS_TOKEN] = access_token self.request_data.body = '' if method in ['POST', 'PUT']: if json is not None: self.request_data.headers[ 'Content-Type'] = 'application/json; charset=utf-8' self.request_data.body = json_util.dumps(json) else: self.request_data.body = '' if data is None else data return real_url
def _parse_uri(self, client): """ 解析请求的 uri :type client: Client :return: """ handler = self.handler endpoint = handler.client.request.endpoint if handler.client.encrypt_type == 'aes': # 如果是加密后的 uri,到这一步的时候已经解密完成 # 需要重新解析 uri, 获取该请求真实的 uri try: _, _, _, uri = handler.request.uri.split('/', 3) if not uri.startswith('/'): uri = '/' + uri handler.client.request.uri = uri except ValueError: raise ClientErrorException(ResultCode.BAD_REQUEST, PromptMessage.INVALID_REQUEST_URI) else: uri = handler.client.request.uri if endpoint.get('is_builtin', False): # 如果是内置的 endpoint, 就没有 forward_url forward_url = None # 寻找匹配的内置 Endpoint Handler key = '%s/%s' % (endpoint['name'], endpoint['version']) builtin_handlers = handler.builtin_endpoints.get(key, []) # 只匹配 uri 的 Path 部分 # 这样 Handlers 那边的正则表达式就可以用 $ 来表示结尾 uri_parsed = urlparse(uri) uri_path = uri_parsed.path for t in builtin_handlers: re_uri, _handler = t match = re.match(re_uri, uri_path) if match: handler.real_api_handler = _handler break else: handler.real_api_handler = None else: # 后端的 API, 需要代理访问 handler.real_api_handler = BackendAPIHandler # 解析要转发的地址 endpoint_url = endpoint.get('url') if endpoint_url is None: raise ServerErrorException( ResultCode.BAD_CLIENT_CONFIG, PromptMessage.NO_ENDPOINT_URL_CONFIG) endpoint_netloc = endpoint.get('netloc') if endpoint_netloc is None: url_parsed = urlparse(endpoint_url) endpoint['netloc'] = url_parsed.netloc if endpoint.get('skip_uri', False): # 判断是否要忽略掉用户传递的uri信息 uri = '' elif endpoint_url.find('?') > 0 and uri == '/': # 有些后端站点,直接定位到查询参数的url,例如/?a=xx,因此后面不能再跟uri, # 而默认会带上/,变成/?a=xx/,因此这里要过滤掉/ uri = '' # TODO uri 合法性校验和修正 if endpoint_url.endswith('/'): forward_url = endpoint_url + uri[1:] else: forward_url = endpoint_url + uri handler.client.request.forward_url = forward_url
def split_url(url): parsed_url = urlparse(url) # Use netloc instead og hostname since hostname is None if URL is relative return parsed_url.scheme == 'https', parsed_url.netloc.lower( ), parsed_url.path
'user': '******', 'non-cookie-auth': '100/hour', 'add-contributor': '10/second', 'create-guid': '1000/hour', 'root-anon-throttle': '1000/hour', 'test-user': '******', 'test-anon': '1/hour', 'send-email': '2/minute', }, } # Settings related to CORS Headers addon: allow API to receive authenticated requests from OSF # CORS plugin only matches based on "netloc" part of URL, so as workaround we add that to the list CORS_ORIGIN_ALLOW_ALL = False CORS_ORIGIN_WHITELIST = ( urlparse(osf_settings.DOMAIN).netloc, osf_settings.DOMAIN, ) # This needs to remain True to allow cross origin requests that are in CORS_ORIGIN_WHITELIST to # use cookies. CORS_ALLOW_CREDENTIALS = True # Set dynamically on app init ORIGINS_WHITELIST = () MIDDLEWARE = ( 'api.base.middleware.DjangoGlobalMiddleware', 'api.base.middleware.CeleryTaskMiddleware', 'api.base.middleware.PostcommitTaskMiddleware', # A profiling middleware. ONLY FOR DEV USE # Uncomment and add "prof" to url params to recieve a profile for that url # 'api.base.middleware.ProfileMiddleware',
def generate_external_export(export_type, username, id_string, export_id=None, options=None, xform=None): """ Generates external export using ONA data through an external service. param: export_type params: username: logged in username params: id_string: xform id_string params: export_id: ID of export object associated with the request param: options: additional parameters required for the lookup. data_id: instance id query: filter_query for custom queries meta: metadata associated with external export token: authentication key required by external service """ data_id = options.get("data_id") filter_query = options.get("query") meta = options.get("meta") token = options.get("token") if xform is None: xform = XForm.objects.get( user__username__iexact=username, id_string__iexact=id_string) user = User.objects.get(username=username) server, name = _get_server_from_metadata(xform, meta, token) # dissect the url parsed_url = urlparse(server) token = parsed_url.path[5:] ser = parsed_url.scheme + '://' + parsed_url.netloc # Get single submission data if data_id: inst = Instance.objects.filter(xform__user=user, xform__id_string=id_string, deleted_at=None, pk=data_id) instances = [inst[0].json if inst else {}] else: instances = query_data(xform, query=filter_query) records = _get_records(instances) status_code = 0 if records and server: try: client = Client(ser) response = client.xls.create(token, json.dumps(records)) if hasattr(client.xls.conn, 'last_response'): status_code = client.xls.conn.last_response.status_code except Exception as e: raise J2XException( u"J2X client could not generate report. Server -> {0}," u" Error-> {1}".format(server, e) ) else: if not server: raise J2XException(u"External server not set") elif not records: raise J2XException( u"No record to export. Form -> {0}".format(id_string) ) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export_options = get_export_options(options) export = Export.objects.create(xform=xform, export_type=export_type, options=export_options) export.export_url = response if status_code == 201: export.internal_status = Export.SUCCESSFUL export.filename = name + '-' + response[5:] if name else response[5:] export.export_url = ser + response else: export.internal_status = Export.FAILED export.save() return export
def sendComment(self, pr_url, text): path = urlparse(unicode2bytes(pr_url)).path payload = {'text': text} return self._http.post(COMMENT_API_URL.format( path=bytes2NativeString(path)), json=payload)
def shardwriter(url, encode=True, pack=True): parsed = urlparse(url) stream = gopen(url, "wb") return tarrecords.TarWriter(stream, encode=encode)
def __init__(self, host=None, port=None, username='******', password=None, verify_ssl=None, config_file=None, use_proxy=False): """Dynamically determine host, port and credential information from current environment :param host: hostname or IP of API server :param port: port of API server :param username: username for basic http authentication (required if token unavailable) :param password: password for basic http authentication (required if token unavailable) :param verify_ssl: whether we trust self-signed certificate """ self.in_pod = bool(os.getenv('KUBERNETES_SERVICE_HOST')) self.session = requests.Session() self.cert = None if use_proxy: self.host = host or "127.0.0.1" self.port = port or "8001" self.url = "http://{ip}:{port}".format(ip=self.host, port=self.port) self.token = None else: session_verify = verify_ssl if self.in_pod and not config_file: self.host = host or 'kubernetes.default' self.port = port or int( os.getenv('KUBERNETES_SERVICE_PORT', 443)) with open(KUBE_TOKEN_PATH) as f: self.token = 'Bearer ' + f.read().strip() swagger_client.configuration.ssl_ca_cert = KUBE_CACRT_PATH if verify_ssl is None: verify_ssl = bool(self.host == KUBE_CACRT_HOST) session_verify = KUBE_CACRT_PATH else: self.port = port or 443 if verify_ssl is None: verify_ssl = False session_verify = False if host and password: self.host = host self.token = urllib3.util.make_headers( basic_auth=username + ':' + password).get('authorization') else: config_path = os.path.expanduser( config_file if config_file else DEFAULT_KUBECONFIG) if not os.path.isfile(config_path): raise ValueError( "Unable to dynamically determine host/credentials from ~/.kube/config" ) with open(config_path) as f: kube_config = yaml.load(f) cred_info = self._parse_config(kube_config) self.host = cred_info['host'] self.token = cred_info.get('token') if 'client-certificate' in cred_info: self.cert = (cred_info['client-certificate'], cred_info['client-key']) p = urlparse('https://{}'.format(self.host)) if p.port: self.port = p.port self.host = p.hostname self.url = 'https://{}:{}'.format(self.host, self.port) # NOTE: options set on the configuration singleton (e.g. verify_ssl), # must be set *before* instantiation of ApiClient for it to take effect. swagger_client.configuration.verify_ssl = verify_ssl self.session.verify = session_verify if self.cert: swagger_client.configuration.cert_file = self.cert[0] swagger_client.configuration.key_file = self.cert[1] self.session.cert = self.cert self.swag_client = swagger_client.ApiClient(self.url) if self.token: self.swag_client.set_default_header('Authorization', self.token) self.session.headers['Authorization'] = self.token # Add modules as attributes of this client self.api = swagger_client.Apiv1Api(self.swag_client) self.version = swagger_client.VersionApi(self.swag_client) self.batchv = swagger_client.Apisbatchv1Api(self.swag_client) self.apisappsv1beta1_api = swagger_client.Apisappsv1beta1Api( self.swag_client) self.extensionsv1beta1 = swagger_client.Apisextensionsv1beta1Api( self.swag_client) self.apisextensionsv1beta1_api = swagger_client.Apisextensionsv1beta1Api( self.swag_client) # Add additional (apps, autoscaling, batch, etc...) for attr in dir(swagger_client): match = re.match(r"^Apis(\w+)Api$", attr) if match: api_class = getattr(swagger_client, attr) api_name = match.group(1) api_instance = api_class(self.swag_client) setattr(self, api_name, api_instance)
def parse_url(url): up = urlparse(url) upport = up.port port = upport if upport else 53 return up.scheme, up.hostname, port
def ParseRegistryURI(uri): url = urlparse.urlparse(uri, scheme="file") return RegistryKeySpec(hive=url.netloc, winreg_hive=getattr(winreg, url.netloc), path=url.path.replace("/", "\\").lstrip("\\"))
def movie_list_request(base_url, port, api_key): parsedurl = urlparse(base_url) log.debug('Received movie list request') return '%s://%s:%s%s/api/%s/movie.list?status=active' % ( parsedurl.scheme, parsedurl.netloc, port, parsedurl.path, api_key)