def without_protocol(url_): """Extract the domain from a URL.""" parsed = utils.urlparse(url_) if parsed.scheme in ('about', 'chrome'): return url_ else: return parsed.netloc
def strip_protocol(url_): """Strip protocol from urls with exceptions for chrome:// and about:.""" parsed = utils.urlparse(url_) if parsed.scheme in ('about', 'chrome'): return url_ else: return parsed.netloc
def sites_url(context, form, url=None, **kwargs): """Return the current form values as URL parameters. Values are taken from the given form and can be overriden using kwargs. This is used to modify parts of a query without losing search context. The 'page' is always reset if not explicitly given. Parameters are only included if the values differ from the default. """ parameters = form.cleaned_data.copy() # page is reset on every change of search for name in form.cleaned_data.keys(): if name == 'page' or parameters[name] == FIELD_DEFS[name].default: del parameters[name] for name, value in kwargs.iteritems(): parameters[name] = value if url or context.get('site'): # single site URL _baseurl = url or context['site'].url _parsed = utils.urlparse(_baseurl) parts = [reverse('single_site', args=[_parsed.scheme, _parsed.netloc])] if 'q' in parameters: del parameters['q'] else: # regular sites search URL parts = [reverse("website_issues")] if len(parameters): parts.extend(["?", urlencode(parameters)]) return ''.join(parts)
def domain(url_): """Remove the protocol from a sites URL (path has been removed already)""" parsed = utils.urlparse(url_) if parsed.scheme == 'chrome': return url_[len('chrome://'):] else: return parsed.netloc
def init_app(self, app): self.config = app.config scheme = None mailer_uri = self.config.get("MAIL_URI") if mailer_uri: mailer_uri = utils.urlparse(mailer_uri) scheme = mailer_uri.scheme hostname = mailer_uri.hostname # Using ses-mailer if "ses" in scheme.lower(): self.provider = "SES" access_key = mailer_uri.username or app.config.get("AWS_ACCESS_KEY_ID") secret_key = mailer_uri.password or app.config.get("AWS_SECRET_ACCESS_KEY") region = hostname or self.config.get("AWS_REGION", "us-east-1") self.mail = ses_mailer.Mail(aws_access_key_id=access_key, aws_secret_access_key=secret_key, region=region, sender=self.config.get("MAIL_SENDER"), reply_to=self.config.get("MAIL_REPLY_TO"), template=self.config.get("MAIL_TEMPLATE"), template_context=self.config.get("MAIL_TEMPLATE_CONTEXT")) # SMTP will use flask-mail elif "smtp" in scheme.lower(): self.provider = "SMTP" class _App(object): config = { "MAIL_SERVER": mailer_uri.hostname, "MAIL_USERNAME": mailer_uri.username, "MAIL_PASSWORD": mailer_uri.password, "MAIL_PORT": mailer_uri.port, "MAIL_USE_TLS": True if "tls" in mailer_uri.scheme else False, "MAIL_USE_SSL": True if "ssl" in mailer_uri.scheme else False, "MAIL_DEFAULT_SENDER": app.config.get("MAIL_SENDER"), "TESTING": app.config.get("TESTING"), "DEBUG": app.config.get("DEBUG") } debug = app.config.get("DEBUG") testing = app.config.get("TESTING") _app = _App() self.mail = flask_mail.Mail(app=_app) _ses_mailer = ses_mailer.Mail(template=self.config.get("MAIL_TEMPLATE"), template_context=self.config.get("MAIL_TEMPLATE_CONTEXT")) self._template = _ses_mailer.parse_template else: logging.warning("Mailer Error. Invalid scheme '%s'>" % scheme)
def _session(app): store = None uri = app.config.get("SESSION_URI") if uri: parse_uri = utils.urlparse(uri) scheme = parse_uri.scheme username = parse_uri.username password = parse_uri.password hostname = parse_uri.hostname port = parse_uri.port bucket = parse_uri.path.strip("/") if "redis" in scheme: import redis from simplekv.memory.redisstore import RedisStore conn = redis.StrictRedis.from_url(url=uri) store = RedisStore(conn) elif "s3" in scheme or "google_storage" in scheme: from simplekv.net.botostore import BotoStore import boto if "s3" in scheme: _con_fn = boto.connect_s3 else: _con_fn = boto.connect_gs conn = _con_fn(username, password) _bucket = conn.create_bucket(bucket) store = BotoStore(_bucket) elif "memcache" in scheme: import memcache from simplekv.memory.memcachestore import MemcacheStore host_port = "%s:%s" % (hostname, port) conn = memcache.Client(servers=[host_port]) store = MemcacheStore(conn) elif "sql" in scheme: from simplekv.db.sql import SQLAlchemyStore from sqlalchemy import create_engine, MetaData engine = create_engine(uri) metadata = MetaData(bind=engine) store = SQLAlchemyStore(engine, metadata, 'kvstore') metadata.create_all() else: raise ValueError("Invalid Session Store") if store: flask_kvsession.KVSessionExtension(store, app)
def extract_reactions(self) -> PartialPost: """Fetch share and reactions information with a existing post obtained by `get_posts`. Return a merged post that has some new fields including `reactions`, `w3_fb_url`, `fetched_time`, and reactions fields `LIKE`, `ANGER`, `SORRY`, `WOW`, `LOVE`, `HAHA` if exist. Note that this method will raise one http request per post, use it when you want some more information. Example: ``` for post in get_posts('fanpage'): more_info_post = fetch_share_and_reactions(post) print(more_info_post) ``` """ url = self.post.get('post_url') post_id = self.post.get('post_id') if url: w3_fb_url = utils.urlparse(url)._replace( netloc='www.facebook.com').geturl() resp = self.request(w3_fb_url) for item in self.parse_share_and_reactions(resp.text): data = item['jsmods']['pre_display_requires'][0][3][1][ '__bbox']['result']['data']['feedback'] if data['subscription_target_id'] == post_id: return { 'shares': data['share_count']['count'], 'likes': data['reactors']['count'], 'reactions': { reaction['node']['reaction_type'].lower(): reaction['reaction_count'] for reaction in data['top_reactions']['edges'] }, 'comments': data['comment_count']['total_count'], 'w3_fb_url': data['url'], 'fetched_time': datetime.now(), } return None
def init_app(self, app): delivery_method = app.config.get("ASSETS_DELIVERY_METHOD") if delivery_method and delivery_method.upper() in ["S3", "CDN"]: #with app.app_context(): is_secure = False #request.is_secure if delivery_method.upper() == "CDN": domain = app.config.get("ASSETS_DELIVERY_DOMAIN") if "://" in domain: domain_parsed = utils.urlparse(domain) is_secure = domain_parsed.scheme == "https" domain = domain_parsed.netloc app.config.setdefault("S3_CDN_DOMAIN", domain) app.config["FLASK_ASSETS_USE_S3"] = True app.config["USE_S3"] = True app.config.setdefault("S3_USE_HTTPS", is_secure) app.config["S3_URL_STYLE"] = "path" app.config.setdefault("S3_ONLY_MODIFIED", True) app.config.setdefault("S3_GZIP", True) app.config.setdefault("S3_BUCKET_NAME", app.config.get("AWS_S3_BUCKET_NAME")) super(self.__class__, self).init_app(app)
def protocol(url_): """Extract the protocol from a URL.""" parsed = utils.urlparse(url_) return parsed.scheme
def sites_url(context, url, **kw): _parsed = utils.urlparse(url) base_url = reverse('single_site', args=[_parsed.scheme, _parsed.netloc]) base_url = base_url + '?' + context['request'].META['QUERY_STRING'] return urlparams(base_url, page=None)