def mangle_url(orig_url, url): try: endpoint_url = urlparse.urlparse(url) except Exception as e: utils.unknown("you must provide an endpoint_url in the form" + "<scheme>://<url>/ (%s)\n" % e) scheme = endpoint_url.scheme if scheme is None: utils.unknown("you must provide an endpoint_url in the form" + "<scheme>://<url>/ (%s)\n" % e) catalog_url = urlparse.urlparse(orig_url) port = endpoint_url.port if port is None: if catalog_url.port is None: port = DAEMON_DEFAULT_PORT else: port = catalog_url.port netloc = "%s:%i" % (endpoint_url.hostname, port) url = urlparse.urlunparse([scheme, netloc, catalog_url.path, catalog_url.params, catalog_url.query, catalog_url.fragment]) return url
def auth_from_url(url): auth = None parsed_url = urlparse.urlparse(url).netloc if '@' in parsed_url: auth = parsed_url.split('@')[0].split(':') auth = requests.auth.HTTPBasicAuth(auth[0], auth[1]) return auth
def startupagent(self, sender, **kwargs): if not self.bind_web_address: _log.info('Web server not started.') return import urlparse parsed = urlparse.urlparse(self.bind_web_address) hostname = parsed.hostname port = parsed.port _log.info('Starting web server binding to {}:{}.' \ .format(hostname, port)) self.registeredroutes.append( (re.compile('^/discovery/$'), 'callable', self._get_discovery)) self.registeredroutes.append( (re.compile('^/discovery/allow$'), 'callable', self._allow)) self.registeredroutes.append( (re.compile('^/$'), 'callable', self._redirect_index)) port = int(port) vhome = os.environ.get('VOLTTRON_HOME') logdir = os.path.join(vhome, "log") if not os.path.exists(logdir): os.makedirs(logdir) self.appContainer = WebApplicationWrapper(self, hostname, port) svr = WSGIServer((hostname, port), self.appContainer) self._server_greenlet = gevent.spawn(svr.serve_forever)
def check(when=time.time): collection = connect_to_mongodb() for data in collection.find(): ip = data.get('ip') target_url = data.get('target_url') ip_stamp = format_time_to_timestamp(data.get('insert_time')) has_existed = int(when() - ip_stamp) if has_existed > over_time: diy_header['Host'] = urlparse.urlparse(target_url).netloc _args = { "url": target_url, "diy_header": diy_header, "time_out": 5, "_ip": ip, } _id = ip + '_' + target_url # 调用验证函数 result1, result2 = valid(_args, False) if result1 is None: msg = 'delete ip: [{ip}], target_url is [{target_url}]'.format(ip=ip, target_url=target_url) logger.info(msg) collection.delete_one({'_id': _id}) else: msg = 'update ip: [{ip}], target_url is [{target_url}]'.format(ip=ip, target_url=target_url) logger.info(msg) collection.update({'_id': _id}, {'insert_time': when()})
def __init__(self, string_proxy=None, request_header=None, timeout=90, debuglevel=0, **kwargs): self.debuglevel = debuglevel self.timeout = timeout self.headers = generator_header() # self.session = requesocks.session(headers=self.headers, timeout=timeout) self.session = requests.session() # self.session.headers = self.headers if string_proxy: socket.getaddrinfo = getaddrinfo urlinfo = urlparse.urlparse(string_proxy) if urlinfo.scheme == 'ssh': self.bitvise = Bitvise(urlinfo.hostname, urlinfo.port, username=urlinfo.username, password=urlinfo.password) forwarding_ip, forwarding_port = self.bitvise.start() string_proxy = 'socks5://%s:%s' % (forwarding_ip, forwarding_port) self.session.proxies = { 'http': string_proxy, 'https': string_proxy } self._body = None
def __init__(self, uri, consumer): asyncore.dispatcher_with_send.__init__(self) self.uri = uri self.consumer = consumer # turn the uri into a valid request scheme, host, path, params, query, fragment = urlparse.urlparse(uri) assert scheme == "http", "only supports HTTP requests" try: host, port = string.split(host, ":", 1) port = int(port) except (TypeError, ValueError): port = 80 # default port if not path: path = "/" if params: path = path + ";" + params if query: path = path + "?" + query self.request = "GET %s HTTP/1.0\r\nHost: %s\r\n\r\n" % (path, host) self.host = host self.port = port self.status = None self.header = None self.data = "" # get things going! self.create_socket(socket.AF_INET, socket.SOCK_STREAM) self.connect((host, port))
def step1_get_authorize_url(self, redirect_uri=None): """Returns a URI to redirect to the provider. Args: redirect_uri: string, Either the string 'urn:ietf:wg:oauth:2.0:oob' for a non-web-based application, or a URI that handles the callback from the authorization server. This parameter is deprecated, please move to passing the redirect_uri in via the constructor. Returns: A URI as a string to redirect the user to begin the authorization flow. """ if redirect_uri is not None: logger.warning(( 'The redirect_uri parameter for' 'OAuth2WebServerFlow.step1_get_authorize_url is deprecated. Please' 'move to passing the redirect_uri in via the constructor.')) self.redirect_uri = redirect_uri if self.redirect_uri is None: raise ValueError('The value of redirect_uri must not be None.') query = { 'response_type': 'code', 'client_id': self.client_id, 'redirect_uri': self.redirect_uri, 'scope': self.scope, } query.update(self.params) parts = list(urlparse.urlparse(self.auth_uri)) query.update(dict(parse_qsl( parts[4]))) # 4 is the index of the query part parts[4] = urllib.parse.urlencode(query) return urlparse.urlunparse(parts)
def do_GET(self): # get the query string variables self.urlparser = urlparse.urlparse(self.path) self.query_string = self.urlparser.query self.query_dict = urlparse.parse_qs(self.query_string) # get the url and output_file self.url = self.query_dict.get('url', [ None, ])[0] self.output_file = self.query_dict.get('output_file', [ None, ])[0] # return error if url or output_file are missing if not self.url or not self.output_file: self.handle_404("url and output_file params are required") return None # convert all query objects from list to single items options_dict = {} for k, v in self.query_dict.items(): options_dict[k] = v[0] wkhtp = WKHtmlToPdf(self.url, self.output_file, **options_dict) output_file = wkhtp.render() # send json response if output_file[0]: self.handle_200("The file has been saved", output_file[1]) else: self.handle_500("%s - the file could not be created" % output_file[1])
def getDetailsColumn(self, struct): ''' This will return the contents of the details column parsed out of the structure returned by a getAccountDetails() returned structure. :param struct: the data structure returned by the getAccountDetails() method :type struct: dictionary :return: Returns the contents of the "description" column from the struct that is sent :rtype: str ''' details = None opkey = self.const.resourcekeys_operation reskey = self.const.resourcekeys_Details resdesc = self.const.resourceKeys_description if ((self.const.resourcekeys_operation in struct) and reskey in struct[opkey]) and \ resdesc in struct[opkey][reskey]: # leaving this here in case the logic above was translated # incorrectly # if ((struct.has_key(self.const.resourcekeys_operation)) and \ # struct[opkey].has_key(reskey)) and \ # struct[opkey][reskey].has_key(resdesc): urlFromDetails = struct[opkey][reskey][resdesc] self.logger.debug("url details: %s", urlFromDetails) parsed_uri = urlparse.urlparse(urlFromDetails) details = parsed_uri.netloc self.logger.debug("urlFromDetails: %s", details) return details
def download(self, url, headers, proxy, num_retries, data=None): print 'Downloading:', url request = urllib.request.Request(url, data, headers or {}) opener = self.opener or urllib.request.build_opener() if proxy: proxy_params = {urlparse.urlparse(url).scheme: proxy} opener.add_handler(urllib.ProxyHandler(proxy_params)) try: response = opener.open(request) html = response.read() code = response.code except Exception as e: print 'Download error:', str(e) html = '' if hasattr(e, 'code'): code = e.code if num_retries > 0 and 500 <= code < 600: # retry 5XX HTTP errors return self._get(url, headers, proxy, num_retries - 1, data) else: code = None return {'html': html.decode(encoding="utf-8"), 'code': code}
def write(self, vals): return_object = super(ExternalSaleOrder, self).write(vals) # Fix if 'shopify_landing_site' in vals: if self.shopify_landing_site: # get params params = {} parsed = urlparse.urlparse(self.shopify_landing_site) params2 = parse_qs(parsed.query) if len(params2) > 0: for param2 in params2: params[str(param2)] = str(params2[param2][0]) # landing_url self.landing_url = parsed.path # utm_campaign if 'utm_campaign' in params: self.landing_utm_campaign = params['utm_campaign'] # utm_medium if 'utm_medium' in params: self.landing_utm_medium = params['utm_medium'] # utm_source if 'utm_source' in params: self.landing_utm_source = params['utm_source'] # return return return_object
def is_local_service(name): """ Determine if a service definition describes a service running on the local node. This is true if the service URL is for localhost, matches the machine's name, or ec2 public name """ if name is None: return False if "://" in name: url = urlparse.urlparse(name) if ":" in url.netloc: name = url.netloc.split(":")[0] else: name = url.netloc elif ":" in name: name = name.split(":")[0] if name == "localhost": return True if '.' in name: name = name.split('.')[0] node = socket.getfqdn() if '.' in node: node = node.split('.')[0] if name == node: return True pn = public_name() if pn is not None and pn.split(".")[0] == name: return True return False
def get_pg_connection() -> (Dict, Dict): """ Gets pg connection and cursor from postgres Returns: (Dict, Dict): the pg connection and cursor """ try: postgres_url = os.getenv('POSTGRES_URL') if(postgres_url): result = urlparse.urlparse(postgres_url) username = result.username password = result.password database = result.path[1:] hostname = result.hostname port = result.port pg_conn = psycopg2.connect( database = database, user = username, password = password, host = hostname, port = port ) else: pg_conn = connect(host='postgres-db', user='******', password='******', dbname='ai') pg_cur = pg_conn.cursor(cursor_factory=DictCursor) return pg_conn, pg_cur except Exception as e: traceback.print_exc() logger_service.log_error('DB ERROR', traceback.format_exc()) raise e
def parse(self, response): bs = BeautifulSoup(response.body) category = urlparse.urlparse(response.url).path[1:-5] for info in bs.findAll('div', {'class': 'product-item-info'}): _id = info.find('a')['href'].split('/')[-1].replace('.html', '') description = info.find('strong').text.strip().split( '\n')[0].strip() price = info.find('span', { 'data-price-type': 'finalPrice' }).text.replace('$', '').strip() if info.find('script'): c = info.find('script').contents[0] a, b = re.search( '"priceByWeight":\{"amount":([^,]+), "weight": ([^}]+)', c).groups() p, w = float(a), float(b) price = p / w date = datetime.date.today().isoformat() sale = len( info.find('div', { 'class': 'product-grid-flags' }).find_all()) == 1 # unidad = info.find('span', {'class': 'price-by'}).text.replace('$','').strip() yield PrecioItem(price=price, description=description, _id=_id, date=date, sale_type=sale, category=category)
def send_email_template(template, context, to, email_from=None, html=None, attachments=[]): """ Renders an email template with this format: {% if subject %}Subject{% endif %} {% if message %}Email body{% endif %} context can be a dictionary or a template.Context instance """ if isinstance(context, dict): context = Context(context) if isinstance(to, str): to = [to] if not 'site' in context: from orchestra import settings url = urlparse.urlparse(settings.ORCHESTRA_SITE_URL) context['site'] = { 'name': settings.ORCHESTRA_SITE_NAME, 'scheme': url.scheme, 'domain': url.netloc, } #subject cannot have new lines subject = render_to_string(template, {'subject': True}, context).strip() message = render_to_string(template, {'message': True}, context) msg = EmailMultiAlternatives(subject, message, email_from, to, attachments=attachments) if html: html_message = render_to_string(html, {'message': True}, context) msg.attach_alternative(html_message, "text/html") msg.send()
def index(request): if request.method == 'PUT': try: data = (request.read().decode('utf-8')) url = "http://192.168.0.184?" url += data parsed = urlparse.urlparse(url) tmp = urlparse.parse_qs(parsed.query)['tmp'][0] stat = "activated" time = str(datetime.today().strftime("%Y/%m/%d %H:%M:%S")) if int(tmp) > 28 and int(tmp) < 31: stat = "activated" strength = "(Weak wind mode)" elif int(tmp) >= 31: stat = "activated" strength = "(Strong wind mode)" else: stat = "deactivated" strength = "" in_data = Mytable(tmp=tmp, stat=stat, strength=strength) in_data.save() except: print("Put Error") return HttpResponse("Complete") if request.method == "GET": try: row = Mytable.objects.order_by('id').last() context = {'row': row} tmp = str(row.tmp) strength = row.strength stat = str(row.stat) except: return HttpResponse("Get Error") return render(request, 'project/present.html', context)
def __init__(self, username, password, url=URL_BASE, validity_ms=120000, auto_login=False, proxies=None): """Log-in to the Luminoso API username, password => (str) credentials to access the server validity_ms => milliseconds of validity for signed messages auto_login => remember the credentials and use them when the connection times out (NOT IMPLEMENTED YET) session => requests session to use for queries""" # Store the login parameters self._auto_login = auto_login self.username = username if auto_login else None self.password = password if auto_login else None self.url = url.rstrip('/') parsed = urlparse.urlparse(self.url) self._host = parsed.netloc # Store the validity parameter self._validity_ms = validity_ms # Initialize the requests session self._session = requests.session() if proxies is not None: self._session.proxies = proxies # Fetch session credentials self.login(username, password)
def data_from_type(name, contenttype, content): if contenttype == 'raw': out = content elif contenttype == 'base64': if len(content) == 0: out = "" else: # FIXME: this is pretty bad; if the content is long, then # we'll be storing it all in memory out = base64.b64decode(content) elif contenttype == 'url': # FIXME: this is really bad; if the file content is long, then we'll be # storing it all in memory. We should probably download it to a # temporary file somewhere and reference that later on. url = urlparse.urlparse(content) if url.scheme == "file": with open(url.netloc + url.path) as f: out = "".join(f.readlines()) else: # fetch url, and put it in self.commands[name] fp = tempfile.TemporaryFile() oz.ozutil.http_download_file(content, fp.fileno(), False, None) fp.seek(0) out = "".join(fp.readlines()) fp.close() else: raise oz.OzException.OzException("Type for %s must be 'raw', 'url' or 'base64'" % (name)) return out
def checkValidDomain(endpoints): validDomains = [] invalidDomains = [] validEndpoints = [] for endpoint in endpoints: endpoint = endpoint.strip().strip("\r").strip('"').strip("'") try: parsedUrl = urlparse.urlparse(endpoint) domain = parsedUrl.netloc.split( ":" )[0] # split is to remove hosts of the following form: example.com:80 if domain in validDomains: validEndpoints.append(endpoint) continue elif domain in invalidDomains: continue try: socket.gethostbyname( domain) # Will throw error if name doesn't resolve validDomains.append(domain) validEndpoints.append(endpoint) except: invalidDomains.append(domain) except: # URL parsing error or resolving error continue return validEndpoints
def _get_expires_from_url(url): if not url: return 0 qs = urlparse.urlparse(url) if 'expire' not in qs or not qs['expire']: return 0 return int(qs['expire'][0])
def _login_authorize_get_token(self): required_props = ['access_token', 'token_type'] self.login() self._login_and_authorize() response = self.client.get(self.redirect_url()) query = QueryDict(urlparse.urlparse(response['Location']).query) code = query['code'] response = self.client.post(self.access_token_url(), { 'grant_type': 'authorization_code', 'client_id': self.get_client().client_id, 'client_secret': self.get_client().client_secret, 'code': code}) self.assertEqual(200, response.status_code, response.content) token = json.loads(response.content) for prop in required_props: self.assertIn(prop, token, "Access token response missing " "required property: %s" % prop) return token
def connect(self, mongo_uri): db_name, options = None, {} u = urlparse.urlparse(mongo_uri) if u.scheme == 'file': path = u.path self._options = urlparse.parse_qs(u.query) if u.query else {} path = u.netloc + path self.db = _MongoImportFileSet(path) if 'uniq' in self._options and 'md5' in self._options['uniq']: self._quick_uniq = _IdHashPairs(path) elif u.scheme == 'mongodb': if '?' in u.path and u.query == '': # url didn't parse it properly u.path is '/dbname?options db_name, self._options = u.path.split('?', 1) self._options = urlparse.parse_qs( self._options) if self._options else {} else: db_name = u.path if db_name.startswith('/'): db_name = db_name[1:] # print('Connecting to db %s on %s with options.' % (db_name, mongo_uri, options)) try: mongo = pymongo.Connection(mongo_uri) except: mongo = pymongo.MongoClient(mongo_uri) self.db = mongo[db_name] if 'uniq' in self._options and 'md5' in self._options['uniq']: self._quick_uniq = False else: raise ValueError( "Invalid URI scheme: '%s'. Can only accept 'file' or 'mongodb'" % u.scheme)
def download(url, headers, proxy, num_retries, data=None): print('Downloading:%s'%url) request = request.Request(url, data, headers) opener = request.build_opener() if proxy: proxy_params = {urlparse.urlparse(url).scheme: proxy} opener.add_handler(request.ProxyHandler(proxy_params)) try: response = opener.open(request) html = response.read() code = response.code except request.URLError as e: print('Download error:%s'%e.reason) html = '' if hasattr(e, 'code'): code = e.code if num_retries > 0 and 500 <= code < 600: # retry 5XX HTTP errors return download(url, headers, proxy, num_retries-1, data) else: code =None return html
def do_GET(self): scheme,netloc,path,params,query,fragment = urlparse.urlparse(self.path) upd, displ = 10, '24h' query = parse_qs(query) if 'update' in query: upd = int(query['update'][0]) if 'display' in query: displ = query['display'][0] if path.endswith('.html') or path.endswith('htm'): self.send_webpage(upd,displ) elif path.endswith('.dat'): f=open(datafile,'r') self.reply('text/plain', f.read() ) f.close() elif path.endswith('.png'): f=open(plotfile,'rb') self.reply('image/png', f.read() ) f.close() elif path.startswith('/temp'): self.reply('text/plain', '%.3f'%temp ) elif path.startswith('/max'): self.reply('text/plain', '%.3f'%max ) elif path.startswith('/min'): self.reply('text/plain', '%.3f'%min ) elif path.startswith('/serial'): self.reply('text/plain', serial ) else: self.send_webpage(upd,displ) return
def render(self, request): """ Render a request by forwarding it to the proxied server. Args: request (Request): Incoming request. Returns: not_done (char): Indicator to note request not yet finished. """ # RFC 2616 tells us that we can omit the port if it's the default port, # but we have to provide it otherwise request.content.seek(0, 0) qs = urlparse.urlparse(request.uri)[4] if qs: rest = self.path + '?' + qs else: rest = self.path clientFactory = self.proxyClientFactoryClass( request.method, rest, request.clientproto, request.getAllHeaders(), request.content.read(), request) clientFactory.noisy = False self.reactor.connectTCP(self.host, self.port, clientFactory) # don't trigger traceback if connection is lost before request finish. request.notifyFinish().addErrback( lambda f: logger.log_trace("%s\nCaught errback in webserver.py:75." % f)) return NOT_DONE_YET
def fetchArgs(): if argc == 3: seedList = sys.argv[1].strip('[]"\'').split(',') termList = sys.argv[2].strip('[]"\'').split(',') for i in range(len(seedList)): seedList[i] = seedList[i].strip(' ') if not bool(urlparse.urlparse(seedList[i]).scheme): usage("URL list argument contains an invalid URL.") return None, None for i in range(len(termList)): termList[i] = termList[i].strip(' ') return seedList, termList elif argc == 13: # In the case people enter 12 strings instead of 2 lists seedList = [] termList = [] for i in range(1, 13): arg = sys.argv[i].strip('[]"\', ') if i in (1, 2): seedList.append(arg) else: termList.append(arg) return seedList, termList else: usage( "Please enter two arguments: [list of seed URLS] | [list of ten related terms]" )
def get_kabupaten(self): for prov in self.provinsi: raw_kab = simple_get(prov['url']) html = bs(raw_kab, 'html.parser') div = html.find('div', {'id': 'TabPaneCuaca1'}) table = div.find('table', {'class': 'table-prakicu-provinsi'}) for a in table.findAll('a'): text = a.text link = a['href'] query = urlparse.urlparse(link).query url = os.path.join(os.path.split(CUACA_URL)[0], link) self.kabupaten.append({ "prov": prov['code'], "name": text, "link": link, "url": url, "code": urlparse.parse_qs(query)['AreaID'][0] }) return self.kabupaten
def do_METHOD_Direct(self): scheme, netloc, path, params, query, fragment = urlparse.urlparse(self.path, 'http') try: host, _, port = netloc.rpartition(':') port = int(port) except ValueError: host = netloc port = 80 try: self.log_request(200) idlecall = None sock = self.socket_create_connection((host, port)) self.headers['Connection'] = 'close' data = '%s %s %s\r\n' % (self.command, urlparse.urlunparse(('', '', path, params, query, '')), self.request_version) data += ''.join('%s: %s\r\n' % (k, self.headers[k]) for k in self.headers if not k.startswith('proxy-')) data += '\r\n' content_length = int(self.headers.get('content-length', 0)) if content_length > 0: data += self.rfile.read(content_length) sock.sendall(data) self.socket_forward(self.connection, sock, idlecall=idlecall) except Exception as ex: logging.exception('LocalProxyHandler.do_GET Error, %s', ex) finally: try: sock.close() del sock except: pass
def mirror_entity_image(self, tweet, entity_index, url): response = requests.get(url, allow_redirects=True, timeout=15) if response.status_code != http.client.OK: log.warn("Failed to download image {0}", url) return content_type = response.headers.get('content-type') parsed_url = urlparse.urlparse(url) (_base, extension) = os.path.splitext(parsed_url.path) extension = None if not extension: extensions = [ ext for ext in mimetypes.guess_all_extensions(content_type) if ext != '.jpe' ] extension = extensions[0] if extensions else '' log.debug("Possible mime types: {0}, chose {1}", extensions, extension) filename = "{tweet}-{index}{extension}".format(tweet=tweet.get('id'), index=entity_index, extension=extension) with NamedTemporaryFile(mode='wb', prefix='twoops', delete=True) as fil: fil.write(response.content) fil.flush() new_url = self.upload_image(fil.name, filename, content_type) if new_url: self.record_tweet_image(tweet, new_url)
def sanitize_redirect(host, redirect_to): """ Given the hostname and an untrusted URL to redirect to, this method tests it to make sure it isn't garbage/harmful and returns it, else returns None, similar as how's it done on django.contrib.auth.views. >>> print sanitize_redirect('myapp.com', None) None >>> print sanitize_redirect('myapp.com', '') None >>> print sanitize_redirect('myapp.com', {}) None >>> print sanitize_redirect('myapp.com', 'http://notmyapp.com/path/') None >>> print sanitize_redirect('myapp.com', 'http://myapp.com/path/') http://myapp.com/path/ >>> print sanitize_redirect('myapp.com', '/path/') /path/ """ # Quick sanity check. if not redirect_to: return None # Heavier security check, don't allow redirection to a different host. try: netloc = urlparse.urlparse(redirect_to)[1] except TypeError: # not valid redirect_to value return None if netloc and netloc != host: return None return redirect_to
def download(self): url = self.imageinfo['url'] if not url.startswith('http://'): url = 'http://' + self.site.host + url url = urlparse.urlparse(url) # TODO: query string return self.site.connection.get(url[1], url[2])
def from_request(http_method, http_url, headers=None, parameters=None, query_string=None): """Combines multiple parameter sources.""" if parameters is None: parameters = {} # Headers if headers and 'Authorization' in headers: auth_header = headers['Authorization'] # Check that the authorization header is OAuth. if auth_header[:6] == 'OAuth ': auth_header = auth_header[6:] try: # Get the parameters from the header. header_params = OAuthRequest._split_header(auth_header) parameters.update(header_params) except Exception: raise OAuthError('Unable to parse OAuth parameters from ' 'Authorization header.') # GET or POST query string. if query_string: query_params = OAuthRequest._split_url_string(query_string) parameters.update(query_params) # URL parameters. param_str = urlparse.urlparse(http_url)[4] # query url_params = OAuthRequest._split_url_string(param_str) parameters.update(url_params) if parameters: return OAuthRequest(http_method, http_url, parameters) return None
def mirror_entity_image(self, tweet, entity_index, url): response = requests.get(url, allow_redirects=True, timeout=15) if response.status_code != http.client.OK: log.warn("Failed to download image {0}", url) return content_type = response.headers.get('content-type') parsed_url = urlparse.urlparse(url) (_base, extension) = os.path.splitext(parsed_url.path) extension = None if not extension: extensions = [ext for ext in mimetypes.guess_all_extensions(content_type) if ext != '.jpe'] extension = extensions[0] if extensions else '' log.debug("Possible mime types: {0}, chose {1}", extensions, extension) filename = "{tweet}-{index}{extension}".format(tweet=tweet.get('id'), index=entity_index, extension=extension) with NamedTemporaryFile(mode='wb', prefix='twoops', delete=True) as fil: fil.write(response.content) fil.flush() new_url = self.upload_image(fil.name, filename, content_type) if new_url: self.record_tweet_image(tweet, new_url)
def _get_store(self, uri): if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir scheme = 'file' else: scheme = urlparse.urlparse(uri).scheme store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri)
def url_add_parameters(url, params): """Adds parameters to URL, parameter will be repeated if already present""" if params: fragments = list(urlparse.urlparse(url)) fragments[4] = urllib.urlencode(parse_qsl(fragments[4]) + params.items()) url = urlparse.urlunparse(fragments) return url
def resolveComponents(url): parsed = urlparse.urlparse(url) new_path = posixpath.normpath(parsed.path) if parsed.path.endswith('/'): # Compensate for issue1707768 new_path += '/' cleaned = parsed._replace(path=new_path) return cleaned.geturl()
def make_job_desc_url(self, job_post_id): url_parts = list(urlparse.urlparse(self.url)) query = dict(urlparse.parse_qsl(url_parts[4])) query.update({'postid': job_post_id}) url_parts[4] = urlencode(query) return urlparse.urlunparse(url_parts)
def ZK(hosts, path): if path.startswith("zk://"): path = urlparse.urlparse(path) hosts = path.netloc path = path.path zk = KazooClient(hosts=hosts) zk.start() return zk, path
def getVideoID(url): url_data = urlparse.urlparse(url) query = urlparse.parse_qs(url_data.query) if not "v" in query: print("Video Url is not valid!") sys.exit(1) video = query["v"][0] return video
except ProductAlert.DoesNotExist: messages.error(self.request, _("No alert found")) else: alert.cancel() messages.success(self.request, _("Alert cancelled")) return HttpResponseRedirect( reverse('customer:summary')+'?tab=alerts' ) def get_emails(self, user):
def get_normalized_http_url(self): """Parses the URL and rebuilds it to be scheme://host/path.""" parts = urlparse.urlparse(self.http_url) scheme, netloc, path = parts[:3] # Exclude default port numbers. if scheme == 'http' and netloc[-3:] == ':80': netloc = netloc[:-3] elif scheme == 'https' and netloc[-4:] == ':443': netloc = netloc[:-4] return '%s://%s%s' % (scheme, netloc, path)
def get_callback_url(self): if self.callback and self.verifier: # Append the oauth_verifier. parts = urlparse.urlparse(self.callback) scheme, netloc, path, params, query, fragment = parts[:6] if query: query = '%s&oauth_verifier=%s' % (query, self.verifier) else: query = 'oauth_verifier=%s' % self.verifier return urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) return self.callback
def check_url_scheme(url): """ Input: url string: a url string Output: True if the url contains scheme False otherwise """ parsed = url_p.urlparse(url) if not parsed.scheme: return False return True
def parse_link_rel(url, fn): """ Read through html file ``fn`` downloaded from ``url``, looking for a link tag of the form: <link rel="alternate" type="application/sage" title="currently ignored" href=".../example.sws" /> This function reads ``fn`` looking for such tags and returns a list of dictionaries of the form {'title': from title field in link, 'url': absolute URL to .sws file} for the corresponding ``.sws`` files. Naturally if there are no appropriate link tags found, the returned list is empty. If the HTML parser raises an HTMLParseError, we simply return an empty list. """ from HTMLParser import HTMLParser class GetLinkRelWorksheets(HTMLParser): def __init__(self): HTMLParser.__init__(self) self.worksheets = [] def handle_starttag(self, tag, attrs): if (tag == 'link' and ('rel', 'alternate') in attrs and ('type', 'application/sage') in attrs): self.worksheets.append({'title': [_ for _ in attrs if _[0] == 'title'][0][1], 'url': [_ for _ in attrs if _[0] == 'href'][0][1]}) parser = GetLinkRelWorksheets() with open(fn) as f: try: parser.feed(f.read()) except HTMLParseError: return [] ret = [] for d in parser.worksheets: sws = d['url'] # is that link a relative URL? if not urlparse.urlparse(sws).netloc: # unquote-then-quote to avoid turning %20 into %2520, etc ret.append({'url': urlparse.urljoin(url, urllib.quote(urllib.unquote(sws))), 'title': d['title']}) else: ret.append({'url': sws, 'title': d['title']}) return ret
def add_url_params(url, new_params, concat=True, unique=True): if isinstance(new_params, dict): new_params = [(k, v) for k, v in new_params.iteritems()] url_parts = list(urlparse.urlparse(url)) params = urlparse.parse_qsl(url_parts[4]) params = new_params if not concat else params + new_params if unique: params = dict(params) url_parts[4] = urllib.urlencode(params) return urlparse.urlunparse(url_parts)
def mangle_url(self, url): # This first connection populate the structure we need inside # the object. This does not cost anything if a connection has # already been made. self.check_connection() try: endpoint_url = urlparse.urlparse(url) except Exception as e: utils.unknown("you must provide an endpoint_url in the form" + "<scheme>://<url>/ (%s)" % e) scheme = endpoint_url.scheme if scheme is None: utils.unknown("you must provide an endpoint_url in the form" + "<scheme>://<url>/ (%s)" % e) catalog_url = None try: catalog_url = urlparse.urlparse( self.nova_client.client.management_url) except Exception as e: utils.unknown("unknown error parsing the catalog url : %s" % e) port = endpoint_url.port if port is None: if catalog_url.port is None: port = self.DAEMON_DEFAULT_PORT else: port = catalog_url.port netloc = "%s:%i" % (endpoint_url.hostname, port) url = urlparse.urlunparse([scheme, netloc, catalog_url.path, catalog_url.params, catalog_url.query, catalog_url.fragment]) self.nova_client.client.management_url = url
def clean_url(self): # Check if we already have a valid embed url url = self.cleaned_data['url'] if url.find('http://www.youtube.com/v/') == 0: return url # Parse querystring to find video ID parsed = urlparse.urlparse(url) qs = urlparse.parse_qs(parsed.query) # Check if the video id exists in query string if 'v' not in qs: raise ValidationError('Osoitteesta ei löytynyt videotunnusta.') # All done. Return valid url return 'http://www.youtube.com/v/'+qs['v'][0]+'/'
def cleanup_url(self, value_url, source_url, mark): """ Transform relative URLs into absolute URLs if possible. If the value_url is already absolute, or we don't know the source_url, then return the existing value. If the value_url is relative, and we know the source_url, then try to rewrite it. """ value = urlparse.urlparse(value_url) if value.netloc or not source_url: url = value_url else: url = urlparse.urljoin(source_url, value_url) if url.startswith('//'): url = 'http:' + url # MissingSchema fix if mark: url = url + mark return url
def get_oauth_url(self): """ Returns the URL with OAuth params """ params = {} if "?" in self.url: url = self.url[: self.url.find("?")] for key, value in urlparse.parse_qsl(urlparse.urlparse(self.url).query): params[key] = value else: url = self.url params["oauth_consumer_key"] = self.consumer_key params["oauth_timestamp"] = int(time()) params["oauth_nonce"] = HMAC(str(time() + randint(0, 99999)).encode(), "secret".encode(), sha1).hexdigest() params["oauth_signature_method"] = "HMAC-SHA256" params["oauth_signature"] = self.generate_oauth_signature(params, url) query_string = urlencode(params) return "%s?%s" % (url, query_string)
def get_back_button(context): """ Show back button, custom title available for different urls, for example 'Back to search results', no back button if user came from other site """ request = context.get('request', None) if not request: raise Exception('Cannot get request from context') referrer = request.META.get('HTTP_REFERER', None) if not referrer: return None try: url = urlparse.urlparse(referrer) except: return None if request.get_host() != url.netloc: try: Site.objects.get(domain=url.netloc) except Site.DoesNotExist: # Came from somewhere else, don't show back button: return None try: match = resolve(url.path) except Resolver404: return None # This dict can be extended to link back to other browsing pages titles = { 'search:search': _('Back to search results'), } title = titles.get(match.view_name, None) if title is None: return None return {'url': referrer, 'title': unicode(title), 'match': match}
def read_url_post(url): '''Transform a JSON contained in a file into an equivalent nested python dict. Parameters ---------- url : string where to get the json. Returns ------- dict Python version of the input Note: if the input is a bare array or literal, for example, the output will be of the corresponding type. ''' urlp = urlparse.urlparse(url) main_url = urlparse.urlunsplit( (urlp.scheme, urlp.netloc, urlp.path, '', '')) data = json.dumps(dict(urlparse.parse_qsl(urlp.query))) handler = urllib_request.HTTPHandler() opener = urllib_request.build_opener(handler) request = urllib_request.Request(main_url, data) request.add_header("Content-Type", 'application/json') request.get_method = lambda: 'POST' try: response = opener.open(request) except Exception as e: response = e if response.code == 200: json_string = response.read() else: json_string = response.read() return json.loads(json_string)
def _retry_using_form_auth(self, response, args): adapter = requests.adapters.HTTPAdapter() request = _copy_request(response.request) u = urlparse.urlparse(response.url) url = urlparse.urlunparse([u.scheme, u.netloc, '/login', None, None, None]) auth = {'username': self.username, 'password': self.password} request2 = requests.Request('POST', url, data=auth).prepare() response2 = adapter.send(request2, **args) if response2.status_code == 401: self.log.error('Login failed: Invalid username or password?') return response cookie = response2.headers.get('set-cookie') if cookie is not None: request.headers['Cookie'] = cookie response3 = adapter.send(request, **args) return response3
def do_GET(self): url = urlparse.urlparse(self.path) path = url.path if path.endswith("/"): path += "index.html" localpath = basedir + path if path == "/detect": params = urlparse.parse_qs(url.query) text = chr(params['text'][0], 'utf-8') json.dump(detector.detect(text), self.wfile) elif os.path.exists(localpath): self.send_response(200) if path.endswith(".html"): self.send_header("Content-Type", "text/html; charset=utf-8") elif path.endswith(".js"): self.send_header("Content-Type", "text/javascript; charset=utf-8") self.end_headers() with open(localpath, "rb") as f: self.wfile.write(f.read()) else: self.send_response(404, "Not Found : " + url.path) self.send_header("Expires", "Fri, 31 Dec 2100 00:00:00 GMT") self.end_headers()
def feed(): Meta = FrontEndMeta(Site) Posts = sortPosts(Pages)[:10] # Generate a unique ID for the site. DomainName = URLParse.urlparse(Site.URIPrefix).hostname if hasattr(Config, "SiteID"): Meta.Site.FeedID = SiteID else: Meta.Site.FeedID = Site.URIPrefix + Config.SiteRoot def urlForWithDomain(endpoint, **values): return Site.URIPrefix + flask.url_for(endpoint, **values) for Post in Posts: # Generate unique IDs for each post, and generate the # creation/update time in iso format. CreationTime = Post["created"] CreationTimeISO = dateTime2ISO8601Format(CreationTime) Post.FeedID = "tag:{},{}:{}".format(DomainName, CreationTime.year, CreationTimeISO) Post.CreatedISO = CreationTimeISO if "updated" in Post.meta: UpdateTime = Post["updated"] Post.UpdatedISO = dateTime2ISO8601Format(UpdateTime) else: Post.UpdatedISO = "" # Derefernece the variables in articles. We cannot use # flask.url_for, because we need absolute URLs. Post.body = varReplace(Post.body, {"app_root": App.config["APPLICATION_ROOT"], "url_for": urlForWithDomain}) # Acquire the update time for the site. Meta.Site.Updated = max([p.UpdatedISO for p in Posts] \ + [p.CreatedISO for p in Posts]) Response = flask.make_response(flask.render_template("feed.xml", pages=Posts, meta=Meta)) Response.mimetype = "application/atom+xml" return Response
def connect_endpoint(self, url): """Connect endpoint from URL. `url` socket.io endpoint URL. """ urldata = urlparse.urlparse(url) endpoint = urldata.path conn = self.endpoints.get(endpoint, None) if conn is None: conn_class = self.conn.get_endpoint(endpoint) if conn_class is None: logger.error('There is no handler for endpoint %s' % endpoint) return conn = conn_class(self, endpoint) self.endpoints[endpoint] = conn self.send_message(proto.connect(endpoint)) if conn.on_open(self.info) == False: self.disconnect_endpoint(endpoint)
def do_POST(self): """ what to do is case of POST request """ parsed_path = urlparse.urlparse(self.path) self.serve_content(parsed_path)