def query_hph( method='get', **argvs): data = OrderedDict(sorted(argvs.items(), key=lambda t: t[0])) time_s = int(time.time()) data_str = unquote(urlencode(data)) hash_str = 'private_key={0}&ts={1}&{2}'.format(private_key, time_s, data_str) sign = hashlib.md5(hash_str.encode('utf-8')).hexdigest() data.update({'sign': sign, 'ts': str(time_s)}) par_str = unquote(urlencode(data)) url = build_url(url=URL, qs=par_str) try: response = requests.get(url) response_dict = response.json() except requests.RequestException as e: return False response_code = response_dict.get('code') if response_code == 0: return response_dict['data'] else: print('no info') return False
def proxy_headers(self, proxy): headers = {} username, password = get_auth_from_url(proxy) if username and password: username = unquote(username) password = unquote(password) headers["Proxy-Authorization"] = _basic_auth_str(username, password) return headers
def is_query_modified(response): def params(url): return dict(q.split('=') for q in urlparse(url).query.split('&')) url_params = params(response['original_url']) alt_params = params(response['alternate_url']) if url_params['text'] == alt_params['text']: return False else: print('''Параметры запроса были изменены во время выполнения: отправлено: %s реально обработано: %s''' % (unquote(url_params['text']), unquote(alt_params['text']))) return True
def getVersion(offset_from_latest): r = requests.get(BASE_URL) soup = BeautifulSoup(r.text, 'html.parser') hrefs = list( (x.get('href') for x in soup.find_all(href=lambda x: x and not x.startswith('.')))) return unquote(hrefs[offset_from_latest].strip('/'))
def story_info(self, story_soup): if not story_soup.find('td'): raise CouldNotParse title = story_soup('td', {'class': 'title'})[-1] subtext = story_soup.find_next('tr').find('td', {'class': 'subtext'}) # Dead post if not subtext.find_all('a'): raise CouldNotParse story = Stories() if title.next == ' [dead] ': story.dead = True story.url = '' else: story.url = unquote(title.find('a')['href']) story.title = title.find('a').contents[0] # Check for domain class if title.find('span', {'class': 'comhead'}): story.selfpost = False else: # No domain provided, must be a selfpost story.selfpost = True story.url = '' story.score = int( re.search(r'(\d+) points?', unicode(subtext.find("span"))).group(1)) story.username = subtext.find('a').find(text=True) try: story.comments = int( re.search(r'(\d+) comments?', unicode(subtext.find_all("a")[1])).group(1)) except AttributeError: # Comments are not always shown (old submissions or ones with 0 comments) story.comments = 0 # Unfortunalely HN doesn't show any form timestamp other than "x hours" # meaning that the time scraped is only approximately correct. story.time = utils.parse_time( subtext.find_all("a")[1].previous_sibling + ' ago') # parsedatetime doesn't have any built in support for DST if time.localtime().tm_isdst: story.time = story.time + datetime.timedelta(hours=-1) story.id = int( re.search('item\?id=(\d+)$', subtext.find_all('a')[1]['href']).group(1)) story.cache = timezone.now() username = get_request().session.get('username', None) if username: # Adding vote auth to session userdata = get_request().session.setdefault('userdata', {}).setdefault( username, {}) try: auth_code = re.search(r'&auth=([a-z0-9]+)&whence', story_soup.a['href']).group(1) except (TypeError, AttributeError): auth_code = None userdata.setdefault('votes', {})[str(story.id)] = auth_code get_request().session.modified = True return story
def prepareFailedName(release): """Standardizes release name for failed DB""" fixed = unquote(release) if fixed.endswith(".nzb"): fixed = fixed.rpartition(".")[0] fixed = re.sub(r"[\.\-\+\ ]", "_", fixed) fixed = ss(fixed) return fixed
def on_notify(self, params): notif = self.notify_schema.validate(params) log.info('aliyun live notification: {0}'.format(params)) user_args = urlparse.parse_qs(unquote(notif['usrargs'])) if notif['action'] == 'publish': for cbk in self._on_publish_cbk_list: cbk(params['id'], user_args) elif notif['action'] == 'publish_done': for cbk in self._on_publish_done_list: cbk(params['id'], user_args) else: raise IVRError('Unknown action {0} from aliyun live notification'.format(notif['action']))
def submit_grade(session, attempt_id, is_group_assignment, grade, text, filenames, rubrics): assert isinstance(session, BlackboardSession) if is_group_assignment: url = ('https://%s/webapps/assignment/' % DOMAIN + 'gradeAssignmentRedirector' + '?course_id=%s' % session.course_id + '&groupAttemptId=%s' % attempt_id) else: url = ('https://%s/webapps/assignment/' % DOMAIN + 'gradeAssignmentRedirector' + '?course_id=%s' % session.course_id + '&attempt_id=%s' % attempt_id) form = Form(session, url, './/h:form[@id="currentAttempt_form"]') form.set('grade', str(grade)) form.set('feedbacktext', text) form.set('gradingNotestext', 'Submitted with https://github.com/Mortal/bbfetch') if rubrics: rubric_input = '%s_rubricEvaluation' % attempt_id rubric_data_str = form.get(rubric_input) rubric_data = json.loads(unquote(rubric_data_str)) for rubric_cells, rubric in zip(rubrics, rubric_data['rubrics']): rubric['client_changed'] = True for input_row, row in zip(rubric_cells, rubric['rows']): row['cell_id'] = input_row rubric_data_str = quote(json.dumps(rubric_data)) form.set(rubric_input, rubric_data_str) for i, filename in enumerate(filenames): base = os.path.basename(filename) form.extend([ ('feedbackFiles_attachmentType', 'L'), ('feedbackFiles_fileId', 'new'), ('feedbackFiles_artifactFileId', 'undefined'), ('feedbackFiles_artifactType', 'undefined'), ('feedbackFiles_artifactTypeResourceKey', 'undefined'), ('feedbackFiles_linkTitle', base), ]) with open(filename, 'rb') as fp: fdata = fp.read() form.files.append(('feedbackFiles_LocalFile%d' % i, (base, fdata))) if is_group_assignment: post_url = ( 'https://%s/webapps/assignment//gradeGroupAssignment/submit' % DOMAIN) else: post_url = ('https://%s/webapps/assignment//gradeAssignment/submit' % DOMAIN) response = form.submit(post_url) form.require_success_message(response)
def story_info(self, story_soup): if not story_soup.find('td'): raise CouldNotParse title = story_soup('td', {'class': 'title'})[-1] subtext = story_soup.find_next('tr').find('td', {'class': 'subtext'}) # Dead post if not subtext.find_all('a'): raise CouldNotParse story = Stories() if title.next == ' [dead] ': story.dead = True story.url = '' else: story.url = unquote(title.find('a')['href']) story.title = title.find('a').contents[0] # Check for domain class if title.find('span', {'class': 'comhead'}): story.selfpost = False else: # No domain provided, must be a selfpost story.selfpost = True story.url = '' story.score = int(re.search(r'(\d+) points?', unicode(subtext.find("span"))).group(1)) story.username = subtext.find('a').find(text=True) try: story.comments = int(re.search(r'(\d+) comments?', unicode(subtext.find_all("a")[1])).group(1)) except AttributeError: # Comments are not always shown (old submissions or ones with 0 comments) story.comments = 0 # Unfortunalely HN doesn't show any form timestamp other than "x hours" # meaning that the time scraped is only approximately correct. story.time = utils.parse_time(subtext.find_all("a")[1].previous_sibling + ' ago') # parsedatetime doesn't have any built in support for DST if time.localtime().tm_isdst: story.time = story.time + datetime.timedelta(hours=-1) story.id = int(re.search('item\?id=(\d+)$', subtext.find_all('a')[1]['href']).group(1)) story.cache = timezone.now() username = get_request().session.get('username', None) if username: # Adding vote auth to session userdata = get_request().session.setdefault('userdata', {}).setdefault(username, {}) try: auth_code = re.search(r'&auth=([a-z0-9]+)&whence', story_soup.a['href']).group(1) except (TypeError, AttributeError): auth_code = None userdata.setdefault('votes', {})[str(story.id)] = auth_code get_request().session.modified = True return story
def runtest(self): url = self.target if ':' in url: response = self.fetch_with_retries(url) if self.parent.check_anchors and '#' in url: anchor = url.split('#')[1] if anchor and "html" in response.headers.get("Content-Type"): parsed = html5lib.parse(response.content, namespaceHTMLElements=False) return self.handle_anchor(parsed, anchor) else: if url.startswith('/'): raise BrokenLinkError(url, "absolute path link") # relative URL anchor = None if '?' in url: url = url.split('?')[0] if '#' in url: url, anchor = url.split('#') if not url and anchor: if self.parent.check_anchors: self.handle_anchor(self.parsed, anchor) return url_path = unquote(url).replace('/', os.path.sep) dirpath = self.fspath.dirpath() exists = False for ext in supported_extensions: rel_path = url_path.replace('.html', ext) target_path = dirpath.join(rel_path) if target_path.exists(): exists = True # only check anchors in html for now if ext == ".html" and anchor and self.parent.check_anchors: with target_path.open() as fpt: parsed = html5lib.parse( fpt, namespaceHTMLElements=False) return self.handle_anchor(parsed, anchor) break if not exists: target_path = dirpath.join(url_path) raise BrokenLinkError(url, "No such file: %s" % target_path)
def handle(self, connection, headers_only=False): with self.lock: if headers_only: connection.send_response(200) connection.send_header('Content-Type', 'application/x-mpegurl') connection.send_header('Connection', 'close') connection.end_headers() return params = { k:[v] for k,v in (unquote(x).split('=') for x in [s2 for s1 in connection.query.split('&') for s2 in s1.split(';')] if '=' in x) } fmt = params['fmt'][0] if 'fmt' in params else None exported = self.createPlaylist(connection.headers['Host'], connection.reqtype, fmt).encode('utf-8') connection.send_response(200) connection.send_header('Content-Type', 'audio/mpegurl; charset=utf-8') connection.send_header('Content-Length', str(len(exported))) connection.send_header('Connection', 'close') connection.end_headers() connection.wfile.write(exported)
def gen_str_to_sign(self, req): """Generate string to sign using giving prepared request""" url = urlsplit(req.url) bucket_name = url.netloc.split(".", 1)[0] logger.debug(req.headers.items()) ucloud_headers = [(k, v.strip()) for k, v in sorted(req.headers.lower_items()) if k.startswith("x-ucloud-")] canonicalized_headers = "\n".join( ["{0}:{1}".format(k, v) for k, v in ucloud_headers]) canonicalized_resource = "/{0}{1}".format(bucket_name, unquote(url.path)) str_to_sign = "\n".join([ req.method, req.headers.get("content-md5", ""), req.headers.get("content-type", ""), req.headers.get("date", self._expires), canonicalized_headers + canonicalized_resource ]) return str_to_sign
def _update_library(self, host=None, showName=None): # pylint: disable=too-many-locals, too-many-return-statements """Handles updating KODI host via HTTP API Attempts to update the KODI video library for a specific tv show if passed, otherwise update the whole library if enabled. Args: host: KODI webserver host:port showName: Name of a TV show to specifically target the library update for Returns: Returns True or False """ if not host: logger.log(u'No KODI host passed, aborting update', logger.WARNING) return False logger.log(u"Updating KODI library via HTTP method for host: " + host, logger.DEBUG) # if we're doing per-show if showName: logger.log(u"Updating library in KODI via HTTP method for show " + showName, logger.DEBUG) pathSql = 'select path.strPath from path, tvshow, tvshowlinkpath where ' \ 'tvshow.c00 = "%s" and tvshowlinkpath.idShow = tvshow.idShow ' \ 'and tvshowlinkpath.idPath = path.idPath' % showName # use this to get xml back for the path lookups xmlCommand = { 'command': 'SetResponseFormat(webheader;false;webfooter;false;header;<xml>;footer;</xml>;opentag;<tag>;closetag;</tag>;closefinaltag;false)'} # sql used to grab path(s) sqlCommand = {'command': 'QueryVideoDatabase(%s)' % pathSql} # set output back to default resetCommand = {'command': 'SetResponseFormat()'} # set xml response format, if this fails then don't bother with the rest request = self._send_to_kodi(xmlCommand, host) if not request: return False sqlXML = self._send_to_kodi(sqlCommand, host) request = self._send_to_kodi(resetCommand, host) if not sqlXML: logger.log(u"Invalid response for " + showName + " on " + host, logger.DEBUG) return False encSqlXML = quote(sqlXML, ':\\/<>') try: et = etree.fromstring(encSqlXML) except SyntaxError as e: logger.log(u"Unable to parse XML returned from KODI: " + ex(e), logger.ERROR) return False paths = et.findall('.//field') if not paths: logger.log(u"No valid paths found for " + showName + " on " + host, logger.DEBUG) return False for path in paths: # we do not need it double-encoded, gawd this is dumb unEncPath = unquote(path.text).decode(app.SYS_ENCODING) logger.log(u"KODI Updating " + showName + " on " + host + " at " + unEncPath, logger.DEBUG) updateCommand = {'command': 'ExecBuiltIn', 'parameter': 'KODI.updatelibrary(video, %s)' % unEncPath} request = self._send_to_kodi(updateCommand, host) if not request: logger.log(u"Update of show directory failed on " + showName + " on " + host + " at " + unEncPath, logger.WARNING) return False # sleep for a few seconds just to be sure kodi has a chance to finish each directory if len(paths) > 1: time.sleep(5) # do a full update if requested else: logger.log(u"Doing Full Library KODI update on host: " + host, logger.DEBUG) updateCommand = {'command': 'ExecBuiltIn', 'parameter': 'KODI.updatelibrary(video)'} request = self._send_to_kodi(updateCommand, host) if not request: logger.log(u"KODI Full Library update failed on: " + host, logger.WARNING) return False return True
def connect(self): sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.settimeout(self.timeout) socket_path = unquote(urlparse(self.unix_socket_url).netloc) sock.connect(socket_path) self.sock = sock
def get_graph_from_request(api): """Process the GET request returning the filtered graph :param DictionaryService api: The dictionary service :return: graph: A BEL graph :rtype: pybel.BELGraph """ network_id = request.args.get(GRAPH_ID) if network_id is not None: network_id = int(network_id) if network_id == 0: network_id = None seed_method = request.args.get(SEED_TYPE) if seed_method and seed_method not in SEED_TYPES: raise ValueError('Invalid seed method: {}'.format(seed_method)) if seed_method and seed_method == SEED_TYPE_PROVENANCE: seed_data = {} authors = request.args.getlist(SEED_DATA_AUTHORS) if authors: seed_data['authors'] = [unquote(author) for author in authors] pmids = request.args.getlist(SEED_DATA_PMIDS) if pmids: seed_data['pmids'] = pmids elif seed_method: seed_data = request.args.getlist(SEED_DATA_NODES) seed_data = [api.decode_node(h) for h in seed_data] else: seed_data = None expand_nodes = request.args.get(APPEND_PARAM) remove_nodes = request.args.get(REMOVE_PARAM) filters = request.args.getlist(FILTERS) filter_pathologies = request.args.get(FILTER_PATHOLOGIES) if filter_pathologies and filter_pathologies in {'True', True}: filter_pathologies = True if expand_nodes: expand_nodes = [api.decode_node(h) for h in expand_nodes.split(',')] if remove_nodes: remove_nodes = [api.decode_node(h) for h in remove_nodes.split(',')] annotations = { k: request.args.getlist(k) for k in request.args if k not in BLACK_LIST } graph = api.query(network_id=network_id, seed_method=seed_method, seed_data=seed_data, expand_nodes=expand_nodes, remove_nodes=remove_nodes, filters=filters, filter_pathologies=filter_pathologies, **annotations) return graph
def handle(self, connection, headers_only=False): P2pproxy.logger.debug('Handling request') hostport = connection.headers['Host'] self.params = { k: [v] for k, v in (unquote(x).split('=') for x in [ s2 for s1 in connection.query.split('&') for s2 in s1.split(';') ] if '=' in x) } # /channels/ branch if connection.reqtype in ('channels', 'channels.m3u'): if connection.path.endswith('play'): # /channels/play?id=[id] channel_id = self.get_param('id') if channel_id is None: # /channels/play?id=&_=[epoch timestamp] is Torrent-TV widget proxy check # P2pProxy simply closes connection on this request sending Server header, so do we if self.get_param('_'): P2pproxy.logger.debug('Status check') connection.send_response(200) connection.send_header('Access-Control-Allow-Origin', '*') connection.send_header('Connection', 'close') connection.send_header('Content-Type', 'text/plain;charset=utf-8') connection.send_header( 'Server', 'P2pProxy/1.0.4.4 HTTPAceProxy') connection.wfile.write('\r\n') return else: connection.dieWithError(400, 'Bad request') # Bad request return if headers_only: connection.send_response(200) connection.send_header('Content-Type', 'video/mpeg') connection.end_headers() return stream_type, stream, translations_list = self.api.stream_source( channel_id) name = logo = '' for channel in translations_list: if channel.getAttribute('id') == channel_id: name = channel.getAttribute('name') logo = channel.getAttribute('logo') if logo != '' and config.fullpathlogo: logo = P2pproxy.TTVU + logo break if stream_type not in ('torrent', 'contentid'): connection.dieWithError( 404, 'Unknown stream type: %s' % stream_type, logging.ERROR) return elif stream_type == 'torrent': connection.path = '/url/%s/stream.mp4' % quote(stream, '') elif stream_type == 'contentid': connection.path = '/content_id/%s/stream.mp4' % stream connection.splittedpath = connection.path.split('/') connection.reqtype = connection.splittedpath[1].lower() connection.handleRequest(headers_only, name, logo, fmt=self.get_param('fmt')) # /channels/?filter=[filter]&group=[group]&type=m3u elif connection.reqtype == 'channels.m3u' or self.get_param( 'type') == 'm3u': if headers_only: connection.send_response(200) connection.send_header('Content-Type', 'application/x-mpegurl') connection.end_headers() return param_group = self.params.get('group') param_filter = self.get_param('filter') if param_filter is None: param_filter = 'all' # default filter if param_group and 'all' in param_group[0]: param_group = None translations_list = self.api.translations(param_filter) playlistgen = PlaylistGenerator( m3uchanneltemplate=config.m3uchanneltemplate) P2pproxy.logger.debug('Generating requested m3u playlist') for channel in translations_list: group_id = channel.getAttribute('group') if param_group and not group_id in param_group[0]: continue # filter channels by &group=1,2,5... name = channel.getAttribute('name') group = TorrentTvApi.CATEGORIES[int(group_id)].decode( 'UTF-8') cid = channel.getAttribute('id') logo = channel.getAttribute('logo') if logo != '' and config.fullpathlogo: logo = P2pproxy.TTVU + logo fields = { 'name': name, 'id': cid, 'url': cid, 'group': group, 'logo': logo } if channel.getAttribute('epg_id') != '0': fields['tvgid'] = config.tvgid % fields playlistgen.addItem(fields) P2pproxy.logger.debug('Exporting m3u playlist') exported = playlistgen.exportm3u( hostport=hostport, header=config.m3uheadertemplate, fmt=self.get_param('fmt')).encode('utf-8') connection.send_response(200) connection.send_header('Content-Type', 'application/x-mpegurl') connection.send_header('Content-Length', str(len(exported))) connection.end_headers() connection.wfile.write(exported) # /channels/?filter=[filter] else: if headers_only: connection.send_response(200) connection.send_header('Access-Control-Allow-Origin', '*') connection.send_header('Connection', 'close') connection.send_header('Content-Type', 'text/xml;charset=utf-8') connection.end_headers() return param_filter = self.get_param('filter') if param_filter is None: param_filter = 'all' # default filter translations_list = self.api.translations(param_filter, True) P2pproxy.logger.debug('Exporting m3u playlist') connection.send_response(200) connection.send_header('Access-Control-Allow-Origin', '*') connection.send_header('Connection', 'close') connection.send_header('Content-Type', 'text/xml;charset=utf-8') connection.send_header('Content-Length', str(len(translations_list))) connection.end_headers() connection.wfile.write(translations_list) # same as /channels request elif connection.reqtype == 'xbmc.pvr' and connection.path.endswith( 'playlist'): connection.send_response(200) connection.send_header('Access-Control-Allow-Origin', '*') connection.send_header('Connection', 'close') connection.send_header('Content-Type', 'text/xml;charset=utf-8') if headers_only: connection.end_headers() return translations_list = self.api.translations('all', True) connection.send_header('Content-Length', str(len(translations_list))) connection.end_headers() P2pproxy.logger.debug('Exporting m3u playlist') connection.wfile.write(translations_list) # /archive/ branch elif connection.reqtype == 'archive': if connection.path.endswith( ('dates', 'dates.m3u')): # /archive/dates.m3u d = datetime.now() delta = timedelta(days=1) playlistgen = PlaylistGenerator() hostport = connection.headers['Host'] days = int( self.get_param('days')) if 'days' in self.params else 7 suffix = '&suffix=' + self.get_param( 'suffix') if 'suffix' in self.params else '' for i in range(days): dfmt = d.strftime('%d-%m-%Y') url = 'http://%s/archive/playlist/?date=%s%s' % ( hostport, dfmt, suffix) playlistgen.addItem({ 'group': '', 'tvg': '', 'name': dfmt, 'url': url }) d -= delta exported = playlistgen.exportm3u( hostport, empty_header=True, process_url=False, fmt=self.get_param('fmt')).encode('utf-8') connection.send_response(200) connection.send_header('Content-Type', 'application/x-mpegurl') connection.send_header('Content-Length', str(len(exported))) connection.end_headers() connection.wfile.write(exported) return elif connection.path.endswith( ('playlist', 'playlist.m3u')): # /archive/playlist.m3u dates = list() if 'date' in self.params: for d in self.params['date']: dates.append(self.parse_date(d).strftime('%d-%m-%Y')) else: d = datetime.now() delta = timedelta(days=1) days = int( self.get_param('days')) if 'days' in self.params else 7 for i in range(days): dates.append(d.strftime('%d-%m-%Y')) d -= delta connection.send_response(200) connection.send_header('Content-Type', 'application/x-mpegurl') if headers_only: connection.end_headers() return channels_list = self.api.archive_channels() hostport = connection.headers['Host'] playlistgen = PlaylistGenerator() suffix = '&suffix=' + self.get_param( 'suffix') if 'suffix' in self.params else '' for channel in channels_list: epg_id = channel.getAttribute('epg_id') name = channel.getAttribute('name') logo = channel.getAttribute('logo') if logo != '' and config.fullpathlogo: logo = P2pproxy.TTVU + logo for d in dates: n = name + ' (' + d + ')' if len(dates) > 1 else name url = 'http://%s/archive/?type=m3u&date=%s&channel_id=%s%s' % ( hostport, d, epg_id, suffix) playlistgen.addItem({ 'group': name, 'tvg': '', 'name': n, 'url': url, 'logo': logo }) exported = playlistgen.exportm3u( hostport, empty_header=True, process_url=False, fmt=self.get_param('fmt')).encode('utf-8') connection.send_header('Content-Length', str(len(exported))) connection.end_headers() connection.wfile.write(exported) return elif connection.path.endswith('channels'): # /archive/channels connection.send_response(200) connection.send_header('Access-Control-Allow-Origin', '*') connection.send_header('Connection', 'close') connection.send_header('Content-Type', 'text/xml;charset=utf-8') if headers_only: connection.end_headers() else: archive_channels = self.api.archive_channels(True) P2pproxy.logger.debug('Exporting m3u playlist') connection.send_header('Content-Length', str(len(archive_channels))) connection.end_headers() connection.wfile.write(archive_channels) return if connection.path.endswith( 'play'): # /archive/play?id=[record_id] record_id = self.get_param('id') if record_id is None: connection.dieWithError(400, 'Bad request') # Bad request return if headers_only: connection.send_response(200) connection.send_header("Content-Type", "video/mpeg") connection.end_headers() return stream_type, stream = self.api.archive_stream_source(record_id) if stream_type not in ('torrent', 'contentid'): connection.dieWithError( 404, 'Unknown stream type: %s' % stream_type, logging.ERROR) return elif stream_type == 'torrent': connection.path = '/url/%s/stream.mp4' % quote(stream, '') elif stream_type == 'contentid': connection.path = '/content_id/%s/stream.mp4' % stream connection.splittedpath = connection.path.split('/') connection.reqtype = connection.splittedpath[1].lower() connection.handleRequest(headers_only, fmt=self.get_param('fmt')) # /archive/?type=m3u&date=[param_date]&channel_id=[param_channel] elif self.get_param('type') == 'm3u': if headers_only: connection.send_response(200) connection.send_header('Content-Type', 'application/x-mpegurl') connection.end_headers() return playlistgen = PlaylistGenerator() param_channel = self.get_param('channel_id') d = self.get_date_param() if param_channel == '' or param_channel is None: channels_list = self.api.archive_channels() for channel in channels_list: channel_id = channel.getAttribute('epg_id') try: records_list = self.api.records(channel_id, d) channel_name = channel.getAttribute('name') logo = channel.getAttribute('logo') if logo != '' and config.fullpathlogo: logo = P2pproxy.TTVU + logo for record in records_list: name = record.getAttribute('name') record_id = record.getAttribute('record_id') playlistgen.addItem({ 'group': channel_name, 'tvg': '', 'name': name, 'url': record_id, 'logo': logo }) except: P2pproxy.logger.debug( 'Failed to load archive for %s' % channel_id) else: records_list = self.api.records(param_channel, d) channels_list = self.api.archive_channels() P2pproxy.logger.debug('Generating archive m3u playlist') for record in records_list: record_id = record.getAttribute('record_id') channel_id = record.getAttribute('epg_id') name = record.getAttribute('name') d = datetime.fromtimestamp( float( record.getAttribute('time'))).strftime('%H:%M') n = '%s %s' % (d, name) logo = '' for channel in channels_list: if channel.getAttribute('epg_id') == channel_id: channel_name = channel.getAttribute('name') logo = channel.getAttribute('logo') if channel_name != '': name = '(' + channel_name + ') ' + name if logo != '' and config.fullpathlogo: logo = P2pproxy.TTVU + logo playlistgen.addItem({ 'group': channel_name, 'name': n, 'url': record_id, 'logo': logo, 'tvg': '' }) P2pproxy.logger.debug('Exporting m3u playlist') exported = playlistgen.exportm3u( hostport, empty_header=True, archive=True, fmt=self.get_param('fmt')).encode('utf-8') connection.send_response(200) connection.send_header('Content-Type', 'application/x-mpegurl') connection.send_header('Content-Length', str(len(exported))) connection.end_headers() connection.wfile.write(exported) # /archive/?date=[param_date]&channel_id=[param_channel] else: param_date = self.get_param('date') if param_date is None: d = datetime.now() else: try: d = parse_date(param_date) except: return param_channel = self.get_param('channel_id') if param_channel == '' or param_channel is None: connection.dieWithError( 500, 'Got /archive/ request but no channel_id specified!', logging.ERROR) return connection.send_response(200) connection.send_header('Access-Control-Allow-Origin', '*') connection.send_header('Connection', 'close') connection.send_header('Content-Type', 'text/xml;charset=utf-8') if headers_only: connection.end_headers() else: records_list = self.api.records(param_channel, d.strftime('%d-%m-%Y'), True) P2pproxy.logger.debug('Exporting m3u playlist') connection.send_header('Content-Length', str(len(records_list))) connection.end_headers() connection.wfile.write(records_list) # Used to generate logomap for the torrenttv plugin elif connection.reqtype == 'logos': translations_list = self.api.translations('all') last = translations_list[-1] connection.send_response(200) connection.send_header('Content-Type', 'text/plain;charset=utf-8') connection.end_headers() connection.wfile.write("logobase = '" + P2pproxy.TTVU + "'\n") connection.wfile.write("logomap = {\n") for channel in translations_list: name = channel.getAttribute('name').encode('utf-8') logo = channel.getAttribute('logo').encode('utf-8') connection.wfile.write(" u'%s': logobase + '%s'" % (name, logo)) if not channel == last: connection.wfile.write(",\n") else: connection.wfile.write("\n") connection.wfile.write("}\n")
def submit_grade(session, attempt_id, is_group_assignment, grade, text, filenames, rubrics): assert isinstance(session, BlackboardSession) if is_group_assignment: url = ('https://bb.au.dk/webapps/assignment/' + 'gradeAssignmentRedirector' + '?course_id=%s' % session.course_id + '&groupAttemptId=%s' % attempt_id) else: url = ('https://bb.au.dk/webapps/assignment/' + 'gradeAssignmentRedirector' + '?course_id=%s' % session.course_id + '&attempt_id=%s' % attempt_id) # We need to fetch the page to get the nonce response = session.get(url) document = html5lib.parse(response.content, encoding=response.encoding) form = document.find('.//h:form[@id="currentAttempt_form"]', NS) if form is None: raise ParserError("No <form id=currentAttempt_form>", response) fields = (form.findall('.//h:input', NS) + form.findall('.//h:textarea', NS)) data = [ (field.get('name'), form_field_value(field)) for field in fields if field.get('name') ] data_lookup = {k: i for i, (k, v) in enumerate(data)} def data_get(k, *args): if args: d, = args try: return data[data_lookup[k]][1] except KeyError: if args: return d raise def data_set(k, v): try: data[data_lookup[k]] = k, v except KeyError: data_lookup[k] = len(data) data.append((k, v)) def data_extend(kvs): for k, v in kvs: data_lookup[k] = len(data) data.append((k, v)) data_set('grade', str(grade)) data_set('feedbacktext', text) data_set('gradingNotestext', 'Submitted with https://github.com/Mortal/bbfetch') if rubrics: rubric_input = '%s_rubricEvaluation' % attempt_id rubric_data_str = data_get(rubric_input) rubric_data = json.loads(unquote(rubric_data_str)) for rubric_cells, rubric in zip(rubrics, rubric_data['rubrics']): rubric['client_changed'] = True for input_row, row in zip(rubric_cells, rubric['rows']): row['cell_id'] = input_row rubric_data_str = quote(json.dumps(rubric_data)) data_set(rubric_input, rubric_data_str) files = [] for i, filename in enumerate(filenames): base = os.path.basename(filename) data_extend([ ('feedbackFiles_attachmentType', 'L'), ('feedbackFiles_fileId', 'new'), ('feedbackFiles_artifactFileId', 'undefined'), ('feedbackFiles_artifactType', 'undefined'), ('feedbackFiles_artifactTypeResourceKey', 'undefined'), ('feedbackFiles_linkTitle', base), ]) with open(filename, 'rb') as fp: fdata = fp.read() files.append(('feedbackFiles_LocalFile%d' % i, (base, fdata))) if is_group_assignment: post_url = ( 'https://bb.au.dk/webapps/assignment//gradeGroupAssignment/submit') else: post_url = ( 'https://bb.au.dk/webapps/assignment//gradeAssignment/submit') if not files: # Blackboard requires the POST to be # Content-Type: multipart/form-data. # Unfortunately, requests can only make a form-data POST # if it has file-like input in the files list. files = [('dummy', io.StringIO(''))] try: response = session.post(post_url, data=data, files=files) except: logger.exception("data=%r files=%r", data, files) raise document = html5lib.parse(response.content, encoding=response.encoding) badmsg = document.find('.//h:span[@id="badMsg1"]', NS) if badmsg is not None: raise ParserError( "badMsg1: %s" % element_text_content(badmsg), response, 'Post data:\n%s' % pprint.pformat(data), 'Files:\n%s' % pprint.pformat(files)) msg = document.find('.//h:span[@id="goodMsg1"]', NS) if msg is None: raise ParserError( "No goodMsg1 in POST response", response, 'Post data:\n%s' % pprint.pformat(data), 'Files:\n%s' % pprint.pformat(files)) logger.debug("goodMsg1: %s", element_text_content(msg))
def fetch_attempt(session, attempt_id, is_group_assignment): assert isinstance(session, BlackboardSession) if is_group_assignment: url = ('https://bb.au.dk/webapps/assignment/' + 'gradeAssignmentRedirector' + '?course_id=%s' % session.course_id + '&groupAttemptId=%s' % attempt_id) else: url = ('https://bb.au.dk/webapps/assignment/' + 'gradeAssignmentRedirector' + '?course_id=%s' % session.course_id + '&attempt_id=%s' % attempt_id) l = blackboard.slowlog() response = session.get(url) l("Fetching attempt took %.1f s") document = html5lib.parse(response.content, encoding=response.encoding) submission_text = document.find( './/h:div[@id="submissionTextView"]', NS) if submission_text is not None: submission_text = element_to_markdown(submission_text) comments = document.find( './/h:div[@id="currentAttempt_comments"]', NS) if comments is not None: xpath = './/h:div[@class="vtbegenerated"]' comments = [ element_to_markdown(e) for e in comments.findall(xpath, NS) ] if not comments: raise blackboard.ParserError( "Page contains currentAttempt_comments, " + "but it contains no comments", response) comments = '\n\n'.join(comments) files = [] submission_list = document.find( './/h:ul[@id="currentAttempt_submissionList"]', NS) if submission_list is None: raise ParserError("No currentAttempt_submissionList", response) for submission in submission_list: filename = element_text_content(submission) download_button = submission.find( './/h:a[@class="dwnldBtn"]', NS) if download_button is not None: download_link = urljoin( response.url, download_button.get('href')) files.append( dict(filename=filename, download_link=download_link)) else: s = 'currentAttempt_attemptFilesubmissionText' a = submission.find( './/h:a[@id="' + s + '"]', NS) if a is not None: # This <li> is for the submission_text if not submission_text: raise blackboard.ParserError( "%r in file list, but no " % (filename,) + "accompanying submission text contents", response) else: raise blackboard.ParserError( "No download link for file %r" % (filename,), response) score_input = document.find( './/h:input[@id="currentAttempt_grade"]', NS) if score_input is None: score = None else: score = form_field_value(score_input) try: score = float(score) except ValueError: if score: raise blackboard.ParserError( "Couldn't parse currentAttempt_grade: %r" % (score,), response) score = None feedbacktext_input = document.find( './/*[@id="feedbacktext"]', NS) if feedbacktext_input is None: feedback = '' else: feedback = form_field_value(feedbacktext_input) if '<' in feedback: feedback = html_to_markdown(feedback) gradingNotestext_input = document.find( './/*[@id="gradingNotestext"]', NS) if gradingNotestext_input is None: grading_notes = '' else: grading_notes = form_field_value(gradingNotestext_input) feedbackfiles_rows = document.find( './/h:tbody[@id="feedbackFiles_table_body"]', NS) feedbackfiles = [] for i, row in enumerate(feedbackfiles_rows or []): try: link = row.findall('.//h:a', NS)[0] except IndexError: raise blackboard.ParserError( "feedbackFiles_table_body row %s: no link" % i, response) download_link = urljoin( response.url, link.get('href')) filename = element_text_content(link) feedbackfiles.append( dict(filename=filename, download_link=download_link)) rubric_data = None if is_group_assignment: rubric_input = document.find( './/h:input[@id="%s_rubricEvaluation"]' % attempt_id, NS) if rubric_input is not None: rubric_data_str = form_field_value(rubric_input) try: rubric_data = json.loads(unquote(rubric_data_str)) except JSONDecodeError: raise ParserError("Couldn't decode JSON", response) t1 = 'blackboard.platform.gradebook2.GroupAttempt' t2 = 'blackboard.plugin.rubric.api.core.data.EvaluationEntity' if rubric_data['evalDataType'] == t1: if rubric_data['evalEntityId'] != attempt_id: raise ParserError( "evalEntityId is %r, expected %r" % (rubric_data['evalEntityId'], attempt_id), response) elif rubric_data['evalDataType'] == t2: # Seems to indicate an already filled-out rubric pass else: raise ParserError( "Unknown evalDataType %r" % rubric_data['evalDataType'], response) return dict( submission=submission_text, comments=comments, files=files, feedback=feedback, feedbackfiles=feedbackfiles, score=score, grading_notes=grading_notes, rubric_data=rubric_data, )
def _update_library(self, host=None, showName=None): # pylint: disable=too-many-locals, too-many-return-statements """Handles updating KODI host via HTTP API Attempts to update the KODI video library for a specific tv show if passed, otherwise update the whole library if enabled. Args: host: KODI webserver host:port showName: Name of a TV show to specifically target the library update for Returns: Returns True or False """ if not host: logger.log(u'No KODI host passed, aborting update', logger.WARNING) return False logger.log(u"Updating KODI library via HTTP method for host: " + host, logger.DEBUG) # if we're doing per-show if showName: logger.log(u"Updating library in KODI via HTTP method for show " + showName, logger.DEBUG) pathSql = 'select path.strPath from path, tvshow, tvshowlinkpath where ' \ 'tvshow.c00 = "%s" and tvshowlinkpath.idShow = tvshow.idShow ' \ 'and tvshowlinkpath.idPath = path.idPath' % showName # use this to get xml back for the path lookups xmlCommand = { 'command': 'SetResponseFormat(webheader;false;webfooter;false;header;<xml>;footer;</xml>;opentag;<tag>;closetag;</tag>;closefinaltag;false)'} # sql used to grab path(s) sqlCommand = {'command': 'QueryVideoDatabase(%s)' % pathSql} # set output back to default resetCommand = {'command': 'SetResponseFormat()'} # set xml response format, if this fails then don't bother with the rest request = self._send_to_kodi(xmlCommand, host) if not request: return False sqlXML = self._send_to_kodi(sqlCommand, host) request = self._send_to_kodi(resetCommand, host) if not sqlXML: logger.log(u"Invalid response for " + showName + " on " + host, logger.DEBUG) return False encSqlXML = quote(sqlXML, ':\\/<>') try: et = etree.fromstring(encSqlXML) except SyntaxError as e: logger.log(u"Unable to parse XML returned from KODI: " + ex(e), logger.ERROR) return False paths = et.findall('.//field') if not paths: logger.log(u"No valid paths found for " + showName + " on " + host, logger.DEBUG) return False for path in paths: # we do not need it double-encoded, gawd this is dumb unEncPath = unquote(path.text).decode(sickbeard.SYS_ENCODING) logger.log(u"KODI Updating " + showName + " on " + host + " at " + unEncPath, logger.DEBUG) updateCommand = {'command': 'ExecBuiltIn', 'parameter': 'KODI.updatelibrary(video, %s)' % unEncPath} request = self._send_to_kodi(updateCommand, host) if not request: logger.log(u"Update of show directory failed on " + showName + " on " + host + " at " + unEncPath, logger.WARNING) return False # sleep for a few seconds just to be sure kodi has a chance to finish each directory if len(paths) > 1: time.sleep(5) # do a full update if requested else: logger.log(u"Doing Full Library KODI update on host: " + host, logger.DEBUG) updateCommand = {'command': 'ExecBuiltIn', 'parameter': 'KODI.updatelibrary(video)'} request = self._send_to_kodi(updateCommand, host) if not request: logger.log(u"KODI Full Library update failed on: " + host, logger.WARNING) return False return True
def fetch_attempt(session, attempt_id, is_group_assignment): assert isinstance(session, BlackboardSession) if is_group_assignment: url = ('https://%s/webapps/assignment/' % DOMAIN + 'gradeAssignmentRedirector' + '?course_id=%s' % session.course_id + '&groupAttemptId=%s' % attempt_id) else: url = ('https://%s/webapps/assignment/' % DOMAIN + 'gradeAssignmentRedirector' + '?course_id=%s' % session.course_id + '&attempt_id=%s' % attempt_id) l = blackboard.slowlog() response = session.get(url) l("Fetching attempt took %.1f s") document = html5lib.parse(response.content, transport_encoding=response.encoding) currentAttempt_container = document.find('.//h:div[@id="currentAttempt"]', NS) if currentAttempt_container is None: not_yet_submitted = ('This attempt has not yet been submitted and ' + 'is not available to view at present.') if not_yet_submitted in response.text: raise NotYetSubmitted raise blackboard.ParserError('No <div id="currentAttempt">', response=response) submission_text = document.find('.//h:div[@id="submissionTextView"]', NS) if submission_text is not None: submission_text = element_to_markdown(submission_text) comments = document.find('.//h:div[@id="currentAttempt_comments"]', NS) if comments is not None: xpath = './/h:div[@class="vtbegenerated"]' comments = [ element_to_markdown(e) for e in comments.findall(xpath, NS) ] if not comments: raise blackboard.ParserError( "Page contains currentAttempt_comments, " + "but it contains no comments", response) comments = '\n\n'.join(comments) files = [] submission_list = document.find( './/h:ul[@id="currentAttempt_submissionList"]', NS) if submission_list is None: if comments is None and submission_text is None: logger.warning("The submission is completely empty.") elif submission_text is None: logger.warning( "No submission; the student only uploaded a comment.") else: logger.warning("The student only uploaded a text submission.") submission_list = () for submission in submission_list: filename = element_text_content(submission) download_button = submission.find('.//h:a[@class="dwnldBtn"]', NS) if download_button is not None: download_link = urljoin(response.url, download_button.get('href')) files.append(dict(filename=filename, download_link=download_link)) else: s = 'currentAttempt_attemptFilesubmissionText' a = submission.find('.//h:a[@id="' + s + '"]', NS) if a is not None: # This <li> is for the submission_text if not submission_text: raise blackboard.ParserError( "%r in file list, but no " % (filename, ) + "accompanying submission text contents", response) else: raise blackboard.ParserError( "No download link for file %r" % (filename, ), response) score_input = document.find('.//h:input[@id="currentAttempt_grade"]', NS) if score_input is None: score = None else: score = form_field_value(score_input) try: score = float(score) except ValueError: if score: raise blackboard.ParserError( "Couldn't parse currentAttempt_grade: %r" % (score, ), response) score = None feedbacktext_input = document.find('.//*[@id="feedbacktext"]', NS) if feedbacktext_input is None: feedback = '' else: feedback = form_field_value(feedbacktext_input) if '<' in feedback: feedback = html_to_markdown(feedback) gradingNotestext_input = document.find('.//*[@id="gradingNotestext"]', NS) if gradingNotestext_input is None: grading_notes = '' else: grading_notes = form_field_value(gradingNotestext_input) feedbackfiles_rows = document.find( './/h:tbody[@id="feedbackFiles_table_body"]', NS) feedbackfiles = [] for i, row in enumerate(feedbackfiles_rows or []): try: link = row.findall('.//h:a', NS)[0] except IndexError: raise blackboard.ParserError( "feedbackFiles_table_body row %s: no link" % i, response) download_link = urljoin(response.url, link.get('href')) filename = element_text_content(link) feedbackfiles.append( dict(filename=filename, download_link=download_link)) rubric_data = None if is_group_assignment: rubric_input = document.find( './/h:input[@id="%s_rubricEvaluation"]' % attempt_id, NS) if rubric_input is not None: rubric_data_str = form_field_value(rubric_input) try: rubric_data = json.loads(unquote(rubric_data_str)) except JSONDecodeError: raise ParserError("Couldn't decode JSON", response) t1 = 'blackboard.platform.gradebook2.GroupAttempt' t2 = 'blackboard.plugin.rubric.api.core.data.EvaluationEntity' if rubric_data['evalDataType'] == t1: if rubric_data['evalEntityId'] != attempt_id: raise ParserError( "evalEntityId is %r, expected %r" % (rubric_data['evalEntityId'], attempt_id), response) elif rubric_data['evalDataType'] == t2: # Seems to indicate an already filled-out rubric pass else: raise ParserError( "Unknown evalDataType %r" % rubric_data['evalDataType'], response) return dict( submission=submission_text, comments=comments, files=files, feedback=feedback, feedbackfiles=feedbackfiles, score=score, grading_notes=grading_notes, rubric_data=rubric_data, )
def send(self, request, **kwargs): """ Wraps a file, described in request, in a Response object. :param request: The PreparedRequest` being "sent". :returns: a Response object containing the file """ # Check that the method makes sense. Only support GET if request.method != "GET": raise ValueError("Invalid request method %s" % request.method) # Parse the URL url_parts = urlparse(request.url) # Reject URLs with a hostname component if url_parts.netloc and url_parts.netloc != "localhost": raise ValueError( "file: URLs with hostname components are not permitted") resp = Response() # Open the file, translate certain errors into HTTP responses # Use urllib's unquote to translate percent escapes into whatever # they actually need to be try: # Split the path on / (the URL directory separator) and decode any # % escapes in the parts path_parts = [unquote(p) for p in url_parts.path.split('/')] # If os.sep is in any of the parts, someone fed us some shenanigans. # Treat is like a missing file. if any(os.sep in p for p in path_parts): raise IOError(errno.ENOENT, os.strerror(errno.ENOENT)) # Put it the path back together # [0] will be an empty string, so stick os.sep in there to make # the path absolute path_parts[0] = os.sep path = os.path.join(*path_parts) resp.raw = open(path, "rb") except IOError as e: if e.errno == errno.EACCES: resp.status_code = 403 elif e.errno == errno.ENOENT: resp.status_code = 404 else: resp.status_code = 400 # Wrap the error message in a file-like object # The error message will be localized, try to convert the string # representation of the exception into a byte stream resp_str = str(e).encode(locale.nl_langinfo(locale.CODESET)) resp.raw = BytesIO(resp_str) resp.headers['Content-Length'] = len(resp_str) else: resp.status_code = 200 # If it's a regular file, set the Content-Length resp_stat = os.fstat(resp.raw.fileno()) if stat.S_ISREG(resp_stat.st_mode): resp.headers['Content-Length'] = resp_stat.st_size return resp
def send(self, request, **kwargs): """ Wraps a file, described in request, in a Response object. :param request: The PreparedRequest` being "sent". :returns: a Response object containing the file """ # Check that the method makes sense. Only support GET if request.method not in ("GET", "HEAD"): raise ValueError("Invalid request method %s" % request.method) # Parse the URL url_parts = urlparse(request.url) # Reject URLs with a hostname component if url_parts.netloc and url_parts.netloc != "localhost": raise ValueError("file: URLs with hostname components are not permitted") resp = Response() # Open the file, translate certain errors into HTTP responses # Use urllib's unquote to translate percent escapes into whatever # they actually need to be try: # Split the path on / (the URL directory separator) and decode any # % escapes in the parts path_parts = [unquote(p) for p in url_parts.path.split('/')] # Strip out the leading empty parts created from the leading /'s while path_parts and not path_parts[0]: path_parts.pop(0) # If os.sep is in any of the parts, someone fed us some shenanigans. # Treat is like a missing file. if any(os.sep in p for p in path_parts): raise IOError(errno.ENOENT, os.strerror(errno.ENOENT)) # Look for a drive component. If one is present, store it separately # so that a directory separator can correctly be added to the real # path, and remove any empty path parts between the drive and the path. # Assume that a part ending with : or | (legacy) is a drive. if path_parts and (path_parts[0].endswith('|') or path_parts[0].endswith(':')): path_drive = path_parts.pop(0) if path_drive.endswith('|'): path_drive = path_drive[:-1] + ':' while path_parts and not path_parts[0]: path_parts.pop(0) else: path_drive = '' # Try to put the path back together # Join the drive back in, and stick os.sep in front of the path to # make it absolute. path = path_drive + os.sep + os.path.join(*path_parts) # Check if the drive assumptions above were correct. If path_drive # is set, and os.path.splitdrive does not return a drive, it wasn't # reall a drive. Put the path together again treating path_drive # as a normal path component. if path_drive and not os.path.splitdrive(path): path = os.sep + os.path.join(path_drive, *path_parts) # Use io.open since we need to add a release_conn method, and # methods can't be added to file objects in python 2. resp.raw = io.open(path, "rb") resp.raw.release_conn = resp.raw.close except IOError as e: if e.errno == errno.EACCES: resp.status_code = codes.forbidden elif e.errno == errno.ENOENT: resp.status_code = codes.not_found else: resp.status_code = codes.bad_request # Wrap the error message in a file-like object # The error message will be localized, try to convert the string # representation of the exception into a byte stream resp_str = str(e).encode(locale.getpreferredencoding(False)) resp.raw = BytesIO(resp_str) resp.headers['Content-Length'] = len(resp_str) # Add release_conn to the BytesIO object resp.raw.release_conn = resp.raw.close else: resp.status_code = codes.ok # If it's a regular file, set the Content-Length resp_stat = os.fstat(resp.raw.fileno()) if stat.S_ISREG(resp_stat.st_mode): resp.headers['Content-Length'] = resp_stat.st_size return resp
def get_connection(self, url, proxies=None): # proxies, silently ignored path = unquote(urlparse(url).netloc) return HTTPUnixConnectionPool(path)
def send(self, request, **kwargs): """Wraps a file, described in request, in a Response object. :param request: The PreparedRequest` being "sent". :returns: a Response object containing the file """ # Check that the method makes sense. Only support GET if request.method not in ("GET", "HEAD"): raise ValueError("Invalid request method %s" % request.method) # Parse the URL url_parts = urlparse(request.url) # Reject URLs with a hostname component if url_parts.netloc and url_parts.netloc != "localhost": raise ValueError( "file: URLs with hostname components are not permitted") resp = Response() # Give the Response some context. resp.request = request # Open the file, translate certain errors into HTTP responses # Use urllib's unquote to translate percent escapes into whatever # they actually need to be try: # Split the path on / (the URL directory separator) and decode any # % escapes in the parts path_parts = [unquote(p) for p in url_parts.path.split("/")] # Strip out the leading empty parts created from the leading /'s while path_parts and not path_parts[0]: path_parts.pop(0) # If os.sep is in any of the parts, someone fed us some shenanigans. # Treat is like a missing file. if any(os.sep in p for p in path_parts): raise IOError(errno.ENOENT, os.strerror(errno.ENOENT)) # Look for a drive component. If one is present, store it separately # so that a directory separator can correctly be added to the real # path, and remove any empty path parts between the drive and the path. # Assume that a part ending with : or | (legacy) is a drive. if path_parts and (path_parts[0].endswith("|") or path_parts[0].endswith(":")): path_drive = path_parts.pop(0) if path_drive.endswith("|"): path_drive = path_drive[:-1] + ":" while path_parts and not path_parts[0]: path_parts.pop(0) else: path_drive = "" # Try to put the path back together # Join the drive back in, # don't stick os.sep in front of the path to keep it relative path = path_drive + os.path.join(*path_parts) # Check if the drive assumptions above were correct. If path_drive # is set, and os.path.splitdrive does not return a drive, it wasn't # reall a drive. Put the path together again treating path_drive # as a normal path component. if path_drive and not os.path.splitdrive(path): path = os.sep + os.path.join(path_drive, *path_parts) # Use io.open since we need to add a release_conn method, and # methods can't be added to file objects in python 2. resp.raw = io.open(path, "rb") resp.raw.release_conn = resp.raw.close except IOError as e: if e.errno == errno.EACCES: resp.status_code = codes.forbidden elif e.errno == errno.ENOENT: resp.status_code = codes.not_found else: resp.status_code = codes.bad_request # Wrap the error message in a file-like object # The error message will be localized, try to convert the string # representation of the exception into a byte stream resp_str = str(e).encode(locale.getpreferredencoding(False)) resp.raw = io.BytesIO(resp_str) if self._set_content_length: resp.headers["Content-Length"] = len(resp_str) # Add release_conn to the BytesIO object resp.raw.release_conn = resp.raw.close else: resp.status_code = codes.ok resp.url = request.url # If it's a regular file, set the Content-Length resp_stat = os.fstat(resp.raw.fileno()) if stat.S_ISREG(resp_stat.st_mode) and self._set_content_length: resp.headers["Content-Length"] = resp_stat.st_size # set Date in RFC 1123 from file mtime resp.headers["Date"] = formatdate(timeval=resp_stat.st_mtime, localtime=False, usegmt=True) return resp
def send(self, request, **kwargs): """ Wraps a file, described in request, in a Response object. :param request: The PreparedRequest` being "sent". :returns: a Response object containing the file """ # Check that the method makes sense. Only support GET if request.method not in ("GET", "HEAD"): raise ValueError("Invalid request method %s" % request.method) # Parse the URL url_parts = urlparse(request.url) # Make the Windows URLs slightly nicer if is_win32 and url_parts.netloc.endswith(":"): url_parts = url_parts._replace(path="/" + url_parts.netloc + url_parts.path, netloc='') # Reject URLs with a hostname component if url_parts.netloc and url_parts.netloc not in ("localhost", ".", "..", "-"): raise ValueError( "file: URLs with hostname components are not permitted") # If the path is relative update it to be absolute if url_parts.netloc in (".", ".."): pwd = os.path.abspath(url_parts.netloc).replace(os.sep, "/") + "/" if is_win32: # prefix the path with a / in Windows pwd = "/" + pwd url_parts = url_parts._replace( path=urljoin(pwd, url_parts.path.lstrip("/"))) resp = Response() resp.url = request.url # Open the file, translate certain errors into HTTP responses # Use urllib's unquote to translate percent escapes into whatever # they actually need to be try: # If the netloc is - then read from stdin if url_parts.netloc == "-": if is_py3: resp.raw = sys.stdin.buffer else: resp.raw = sys.stdin # make a fake response URL, the current directory resp.url = "file://" + os.path.abspath(".").replace( os.sep, "/") + "/" else: # Split the path on / (the URL directory separator) and decode any # % escapes in the parts path_parts = [unquote(p) for p in url_parts.path.split('/')] # Strip out the leading empty parts created from the leading /'s while path_parts and not path_parts[0]: path_parts.pop(0) # If os.sep is in any of the parts, someone fed us some shenanigans. # Treat is like a missing file. if any(os.sep in p for p in path_parts): raise IOError(errno.ENOENT, os.strerror(errno.ENOENT)) # Look for a drive component. If one is present, store it separately # so that a directory separator can correctly be added to the real # path, and remove any empty path parts between the drive and the path. # Assume that a part ending with : or | (legacy) is a drive. if path_parts and (path_parts[0].endswith('|') or path_parts[0].endswith(':')): path_drive = path_parts.pop(0) if path_drive.endswith('|'): path_drive = path_drive[:-1] + ':' while path_parts and not path_parts[0]: path_parts.pop(0) else: path_drive = '' # Try to put the path back together # Join the drive back in, and stick os.sep in front of the path to # make it absolute. path = path_drive + os.sep + os.path.join(*path_parts) # Check if the drive assumptions above were correct. If path_drive # is set, and os.path.splitdrive does not return a drive, it wasn't # reall a drive. Put the path together again treating path_drive # as a normal path component. if path_drive and not os.path.splitdrive(path): path = os.sep + os.path.join(path_drive, *path_parts) # Use io.open since we need to add a release_conn method, and # methods can't be added to file objects in python 2. resp.raw = io.open(path, "rb") resp.raw.release_conn = resp.raw.close except IOError as e: if e.errno == errno.EACCES: resp.status_code = codes.forbidden elif e.errno == errno.ENOENT: resp.status_code = codes.not_found else: resp.status_code = codes.bad_request # Wrap the error message in a file-like object # The error message will be localized, try to convert the string # representation of the exception into a byte stream resp_str = str(e).encode(locale.getpreferredencoding(False)) resp.raw = BytesIO(resp_str) resp.headers['Content-Length'] = len(resp_str) # Add release_conn to the BytesIO object resp.raw.release_conn = resp.raw.close else: resp.status_code = codes.ok # If it's a regular file, set the Content-Length resp_stat = os.fstat(resp.raw.fileno()) if stat.S_ISREG(resp_stat.st_mode): resp.headers['Content-Length'] = resp_stat.st_size return resp
def send(self, request, **kwargs): """ Wraps a file, described in request, in a Response object. :param request: The PreparedRequest` being "sent". :returns: a Response object containing the file """ # Check that the method makes sense. Only support GET if request.method != "GET": raise ValueError("Invalid request method %s" % request.method) # Parse the URL url_parts = urlparse(request.url) # Reject URLs with a hostname component if url_parts.netloc and url_parts.netloc != "localhost": raise ValueError("file: URLs with hostname components are not permitted") resp = Response() # Open the file, translate certain errors into HTTP responses # Use urllib's unquote to translate percent escapes into whatever # they actually need to be try: # Split the path on / (the URL directory separator) and decode any # % escapes in the parts path_parts = [unquote(p) for p in url_parts.path.split('/')] # If os.sep is in any of the parts, someone fed us some shenanigans. # Treat is like a missing file. if any(os.sep in p for p in path_parts): raise IOError(errno.ENOENT, os.strerror(errno.ENOENT)) # Put it the path back together # [0] will be an empty string, so stick os.sep in there to make # the path absolute path_parts[0] = os.sep path = os.path.join(*path_parts) resp.raw = open(path, "rb") except IOError as e: if e.errno == errno.EACCES: resp.status_code = 403 elif e.errno == errno.ENOENT: resp.status_code = 404 else: resp.status_code = 400 # Wrap the error message in a file-like object # The error message will be localized, try to convert the string # representation of the exception into a byte stream resp_str = str(e).encode(locale.nl_langinfo(locale.CODESET)) resp.raw = BytesIO(resp_str) resp.headers['Content-Length'] = len(resp_str) else: resp.status_code = 200 # If it's a regular file, set the Content-Length resp_stat = os.fstat(resp.raw.fileno()) if stat.S_ISREG(resp_stat.st_mode): resp.headers['Content-Length'] = resp_stat.st_size return resp
def unescape(_str): return unicode(unquote(_str)).encode('utf8')
def latest_version(): r = requests.get(BASE_URL) soup = BeautifulSoup(r.text, 'html.parser') hrefs = (x.get('href') for x in soup.find_all( href=lambda x: x and not x.startswith('.'))) return unquote(max(hrefs)).strip('/')