def without_protocol(url_):
"""Extract the domain from a URL."""
parsed = utils.urlparse(url_)
if parsed.scheme in ('about', 'chrome'):
return url_
else:
return parsed.netloc
def strip_protocol(url_):
"""Strip protocol from urls with exceptions for chrome:// and about:."""
parsed = utils.urlparse(url_)
if parsed.scheme in ('about', 'chrome'):
return url_
else:
return parsed.netloc
def sites_url(context, form, url=None, **kwargs):
"""Return the current form values as URL parameters.
Values are taken from the given form and can be overriden using kwargs.
This is used to modify parts of a query without losing search context.
The 'page' is always reset if not explicitly given.
Parameters are only included if the values differ from the default.
"""
parameters = form.cleaned_data.copy()
# page is reset on every change of search
for name in form.cleaned_data.keys():
if name == 'page' or parameters[name] == FIELD_DEFS[name].default:
del parameters[name]
for name, value in kwargs.iteritems():
parameters[name] = value
if url or context.get('site'):
# single site URL
_baseurl = url or context['site'].url
_parsed = utils.urlparse(_baseurl)
parts = [reverse('single_site', args=[_parsed.scheme, _parsed.netloc])]
if 'q' in parameters:
del parameters['q']
else:
# regular sites search URL
parts = [reverse("website_issues")]
if len(parameters):
parts.extend(["?", urlencode(parameters)])
return ''.join(parts)
def domain(url_):
"""Remove the protocol from a sites URL (path has been removed already)"""
parsed = utils.urlparse(url_)
if parsed.scheme == 'chrome':
return url_[len('chrome://'):]
else:
return parsed.netloc
def init_app(self, app):
self.config = app.config
scheme = None
mailer_uri = self.config.get("MAIL_URI")
if mailer_uri:
mailer_uri = utils.urlparse(mailer_uri)
scheme = mailer_uri.scheme
hostname = mailer_uri.hostname
# Using ses-mailer
if "ses" in scheme.lower():
self.provider = "SES"
access_key = mailer_uri.username or app.config.get("AWS_ACCESS_KEY_ID")
secret_key = mailer_uri.password or app.config.get("AWS_SECRET_ACCESS_KEY")
region = hostname or self.config.get("AWS_REGION", "us-east-1")
self.mail = ses_mailer.Mail(aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region=region,
sender=self.config.get("MAIL_SENDER"),
reply_to=self.config.get("MAIL_REPLY_TO"),
template=self.config.get("MAIL_TEMPLATE"),
template_context=self.config.get("MAIL_TEMPLATE_CONTEXT"))
# SMTP will use flask-mail
elif "smtp" in scheme.lower():
self.provider = "SMTP"
class _App(object):
config = {
"MAIL_SERVER": mailer_uri.hostname,
"MAIL_USERNAME": mailer_uri.username,
"MAIL_PASSWORD": mailer_uri.password,
"MAIL_PORT": mailer_uri.port,
"MAIL_USE_TLS": True if "tls" in mailer_uri.scheme else False,
"MAIL_USE_SSL": True if "ssl" in mailer_uri.scheme else False,
"MAIL_DEFAULT_SENDER": app.config.get("MAIL_SENDER"),
"TESTING": app.config.get("TESTING"),
"DEBUG": app.config.get("DEBUG")
}
debug = app.config.get("DEBUG")
testing = app.config.get("TESTING")
_app = _App()
self.mail = flask_mail.Mail(app=_app)
_ses_mailer = ses_mailer.Mail(template=self.config.get("MAIL_TEMPLATE"),
template_context=self.config.get("MAIL_TEMPLATE_CONTEXT"))
self._template = _ses_mailer.parse_template
else:
logging.warning("Mailer Error. Invalid scheme '%s'>" % scheme)
def _session(app):
store = None
uri = app.config.get("SESSION_URI")
if uri:
parse_uri = utils.urlparse(uri)
scheme = parse_uri.scheme
username = parse_uri.username
password = parse_uri.password
hostname = parse_uri.hostname
port = parse_uri.port
bucket = parse_uri.path.strip("/")
if "redis" in scheme:
import redis
from simplekv.memory.redisstore import RedisStore
conn = redis.StrictRedis.from_url(url=uri)
store = RedisStore(conn)
elif "s3" in scheme or "google_storage" in scheme:
from simplekv.net.botostore import BotoStore
import boto
if "s3" in scheme:
_con_fn = boto.connect_s3
else:
_con_fn = boto.connect_gs
conn = _con_fn(username, password)
_bucket = conn.create_bucket(bucket)
store = BotoStore(_bucket)
elif "memcache" in scheme:
import memcache
from simplekv.memory.memcachestore import MemcacheStore
host_port = "%s:%s" % (hostname, port)
conn = memcache.Client(servers=[host_port])
store = MemcacheStore(conn)
elif "sql" in scheme:
from simplekv.db.sql import SQLAlchemyStore
from sqlalchemy import create_engine, MetaData
engine = create_engine(uri)
metadata = MetaData(bind=engine)
store = SQLAlchemyStore(engine, metadata, 'kvstore')
metadata.create_all()
else:
raise ValueError("Invalid Session Store")
if store:
flask_kvsession.KVSessionExtension(store, app)
def extract_reactions(self) -> PartialPost:
"""Fetch share and reactions information with a existing post obtained by `get_posts`.
Return a merged post that has some new fields including `reactions`, `w3_fb_url`,
`fetched_time`, and reactions fields `LIKE`, `ANGER`, `SORRY`, `WOW`, `LOVE`, `HAHA` if
exist.
Note that this method will raise one http request per post, use it when you want some more
information.
Example:
```
for post in get_posts('fanpage'):
more_info_post = fetch_share_and_reactions(post)
print(more_info_post)
```
"""
url = self.post.get('post_url')
post_id = self.post.get('post_id')
if url:
w3_fb_url = utils.urlparse(url)._replace(
netloc='www.facebook.com').geturl()
resp = self.request(w3_fb_url)
for item in self.parse_share_and_reactions(resp.text):
data = item['jsmods']['pre_display_requires'][0][3][1][
'__bbox']['result']['data']['feedback']
if data['subscription_target_id'] == post_id:
return {
'shares': data['share_count']['count'],
'likes': data['reactors']['count'],
'reactions': {
reaction['node']['reaction_type'].lower():
reaction['reaction_count']
for reaction in data['top_reactions']['edges']
},
'comments': data['comment_count']['total_count'],
'w3_fb_url': data['url'],
'fetched_time': datetime.now(),
}
return None
def init_app(self, app):
delivery_method = app.config.get("ASSETS_DELIVERY_METHOD")
if delivery_method and delivery_method.upper() in ["S3", "CDN"]:
#with app.app_context():
is_secure = False #request.is_secure
if delivery_method.upper() == "CDN":
domain = app.config.get("ASSETS_DELIVERY_DOMAIN")
if "://" in domain:
domain_parsed = utils.urlparse(domain)
is_secure = domain_parsed.scheme == "https"
domain = domain_parsed.netloc
app.config.setdefault("S3_CDN_DOMAIN", domain)
app.config["FLASK_ASSETS_USE_S3"] = True
app.config["USE_S3"] = True
app.config.setdefault("S3_USE_HTTPS", is_secure)
app.config["S3_URL_STYLE"] = "path"
app.config.setdefault("S3_ONLY_MODIFIED", True)
app.config.setdefault("S3_GZIP", True)
app.config.setdefault("S3_BUCKET_NAME", app.config.get("AWS_S3_BUCKET_NAME"))
super(self.__class__, self).init_app(app)
def protocol(url_):
"""Extract the protocol from a URL."""
parsed = utils.urlparse(url_)
return parsed.scheme
def protocol(url_):
"""Extract the protocol from a URL."""
parsed = utils.urlparse(url_)
return parsed.scheme
def sites_url(context, url, **kw):
_parsed = utils.urlparse(url)
base_url = reverse('single_site', args=[_parsed.scheme, _parsed.netloc])
base_url = base_url + '?' + context['request'].META['QUERY_STRING']
return urlparams(base_url, page=None)
