def query_hph( method='get', **argvs):
data = OrderedDict(sorted(argvs.items(), key=lambda t: t[0]))
time_s = int(time.time())
data_str = unquote(urlencode(data))
hash_str = 'private_key={0}&ts={1}&{2}'.format(private_key,
time_s,
data_str)
sign = hashlib.md5(hash_str.encode('utf-8')).hexdigest()
data.update({'sign': sign, 'ts': str(time_s)})
par_str = unquote(urlencode(data))
url = build_url(url=URL, qs=par_str)
try:
response = requests.get(url)
response_dict = response.json()
except requests.RequestException as e:
return False
response_code = response_dict.get('code')
if response_code == 0:
return response_dict['data']
else:
print('no info')
return False
def proxy_headers(self, proxy):
headers = {}
username, password = get_auth_from_url(proxy)
if username and password:
username = unquote(username)
password = unquote(password)
headers["Proxy-Authorization"] = _basic_auth_str(username, password)
return headers
def is_query_modified(response):
def params(url):
return dict(q.split('=') for q in urlparse(url).query.split('&'))
url_params = params(response['original_url'])
alt_params = params(response['alternate_url'])
if url_params['text'] == alt_params['text']:
return False
else:
print('''Параметры запроса были изменены во время выполнения:
отправлено: %s
реально обработано: %s''' %
(unquote(url_params['text']), unquote(alt_params['text'])))
return True
def getVersion(offset_from_latest):
r = requests.get(BASE_URL)
soup = BeautifulSoup(r.text, 'html.parser')
hrefs = list(
(x.get('href')
for x in soup.find_all(href=lambda x: x and not x.startswith('.'))))
return unquote(hrefs[offset_from_latest].strip('/'))
def story_info(self, story_soup):
if not story_soup.find('td'):
raise CouldNotParse
title = story_soup('td', {'class': 'title'})[-1]
subtext = story_soup.find_next('tr').find('td', {'class': 'subtext'})
# Dead post
if not subtext.find_all('a'):
raise CouldNotParse
story = Stories()
if title.next == ' [dead] ':
story.dead = True
story.url = ''
else:
story.url = unquote(title.find('a')['href'])
story.title = title.find('a').contents[0]
# Check for domain class
if title.find('span', {'class': 'comhead'}):
story.selfpost = False
else:
# No domain provided, must be a selfpost
story.selfpost = True
story.url = ''
story.score = int(
re.search(r'(\d+) points?',
unicode(subtext.find("span"))).group(1))
story.username = subtext.find('a').find(text=True)
try:
story.comments = int(
re.search(r'(\d+) comments?',
unicode(subtext.find_all("a")[1])).group(1))
except AttributeError:
# Comments are not always shown (old submissions or ones with 0 comments)
story.comments = 0
# Unfortunalely HN doesn't show any form timestamp other than "x hours"
# meaning that the time scraped is only approximately correct.
story.time = utils.parse_time(
subtext.find_all("a")[1].previous_sibling + ' ago')
# parsedatetime doesn't have any built in support for DST
if time.localtime().tm_isdst:
story.time = story.time + datetime.timedelta(hours=-1)
story.id = int(
re.search('item\?id=(\d+)$',
subtext.find_all('a')[1]['href']).group(1))
story.cache = timezone.now()
username = get_request().session.get('username', None)
if username:
# Adding vote auth to session
userdata = get_request().session.setdefault('userdata',
{}).setdefault(
username, {})
try:
auth_code = re.search(r'&auth=([a-z0-9]+)&whence',
story_soup.a['href']).group(1)
except (TypeError, AttributeError):
auth_code = None
userdata.setdefault('votes', {})[str(story.id)] = auth_code
get_request().session.modified = True
return story
def prepareFailedName(release):
"""Standardizes release name for failed DB"""
fixed = unquote(release)
if fixed.endswith(".nzb"):
fixed = fixed.rpartition(".")[0]
fixed = re.sub(r"[\.\-\+\ ]", "_", fixed)
fixed = ss(fixed)
return fixed
def on_notify(self, params):
notif = self.notify_schema.validate(params)
log.info('aliyun live notification: {0}'.format(params))
user_args = urlparse.parse_qs(unquote(notif['usrargs']))
if notif['action'] == 'publish':
for cbk in self._on_publish_cbk_list:
cbk(params['id'], user_args)
elif notif['action'] == 'publish_done':
for cbk in self._on_publish_done_list:
cbk(params['id'], user_args)
else:
raise IVRError('Unknown action {0} from aliyun live notification'.format(notif['action']))
def submit_grade(session, attempt_id, is_group_assignment, grade, text,
filenames, rubrics):
assert isinstance(session, BlackboardSession)
if is_group_assignment:
url = ('https://%s/webapps/assignment/' % DOMAIN +
'gradeAssignmentRedirector' +
'?course_id=%s' % session.course_id +
'&groupAttemptId=%s' % attempt_id)
else:
url = ('https://%s/webapps/assignment/' % DOMAIN +
'gradeAssignmentRedirector' +
'?course_id=%s' % session.course_id +
'&attempt_id=%s' % attempt_id)
form = Form(session, url, './/h:form[@id="currentAttempt_form"]')
form.set('grade', str(grade))
form.set('feedbacktext', text)
form.set('gradingNotestext',
'Submitted with https://github.com/Mortal/bbfetch')
if rubrics:
rubric_input = '%s_rubricEvaluation' % attempt_id
rubric_data_str = form.get(rubric_input)
rubric_data = json.loads(unquote(rubric_data_str))
for rubric_cells, rubric in zip(rubrics, rubric_data['rubrics']):
rubric['client_changed'] = True
for input_row, row in zip(rubric_cells, rubric['rows']):
row['cell_id'] = input_row
rubric_data_str = quote(json.dumps(rubric_data))
form.set(rubric_input, rubric_data_str)
for i, filename in enumerate(filenames):
base = os.path.basename(filename)
form.extend([
('feedbackFiles_attachmentType', 'L'),
('feedbackFiles_fileId', 'new'),
('feedbackFiles_artifactFileId', 'undefined'),
('feedbackFiles_artifactType', 'undefined'),
('feedbackFiles_artifactTypeResourceKey', 'undefined'),
('feedbackFiles_linkTitle', base),
])
with open(filename, 'rb') as fp:
fdata = fp.read()
form.files.append(('feedbackFiles_LocalFile%d' % i, (base, fdata)))
if is_group_assignment:
post_url = (
'https://%s/webapps/assignment//gradeGroupAssignment/submit' %
DOMAIN)
else:
post_url = ('https://%s/webapps/assignment//gradeAssignment/submit' %
DOMAIN)
response = form.submit(post_url)
form.require_success_message(response)
def story_info(self, story_soup):
if not story_soup.find('td'):
raise CouldNotParse
title = story_soup('td', {'class': 'title'})[-1]
subtext = story_soup.find_next('tr').find('td', {'class': 'subtext'})
# Dead post
if not subtext.find_all('a'):
raise CouldNotParse
story = Stories()
if title.next == ' [dead] ':
story.dead = True
story.url = ''
else:
story.url = unquote(title.find('a')['href'])
story.title = title.find('a').contents[0]
# Check for domain class
if title.find('span', {'class': 'comhead'}):
story.selfpost = False
else:
# No domain provided, must be a selfpost
story.selfpost = True
story.url = ''
story.score = int(re.search(r'(\d+) points?', unicode(subtext.find("span"))).group(1))
story.username = subtext.find('a').find(text=True)
try:
story.comments = int(re.search(r'(\d+) comments?', unicode(subtext.find_all("a")[1])).group(1))
except AttributeError:
# Comments are not always shown (old submissions or ones with 0 comments)
story.comments = 0
# Unfortunalely HN doesn't show any form timestamp other than "x hours"
# meaning that the time scraped is only approximately correct.
story.time = utils.parse_time(subtext.find_all("a")[1].previous_sibling + ' ago')
# parsedatetime doesn't have any built in support for DST
if time.localtime().tm_isdst:
story.time = story.time + datetime.timedelta(hours=-1)
story.id = int(re.search('item\?id=(\d+)$', subtext.find_all('a')[1]['href']).group(1))
story.cache = timezone.now()
username = get_request().session.get('username', None)
if username:
# Adding vote auth to session
userdata = get_request().session.setdefault('userdata', {}).setdefault(username, {})
try:
auth_code = re.search(r'&auth=([a-z0-9]+)&whence', story_soup.a['href']).group(1)
except (TypeError, AttributeError):
auth_code = None
userdata.setdefault('votes', {})[str(story.id)] = auth_code
get_request().session.modified = True
return story
def runtest(self):
url = self.target
if ':' in url:
response = self.fetch_with_retries(url)
if self.parent.check_anchors and '#' in url:
anchor = url.split('#')[1]
if anchor and "html" in response.headers.get("Content-Type"):
parsed = html5lib.parse(response.content,
namespaceHTMLElements=False)
return self.handle_anchor(parsed, anchor)
else:
if url.startswith('/'):
raise BrokenLinkError(url, "absolute path link")
# relative URL
anchor = None
if '?' in url:
url = url.split('?')[0]
if '#' in url:
url, anchor = url.split('#')
if not url and anchor:
if self.parent.check_anchors:
self.handle_anchor(self.parsed, anchor)
return
url_path = unquote(url).replace('/', os.path.sep)
dirpath = self.fspath.dirpath()
exists = False
for ext in supported_extensions:
rel_path = url_path.replace('.html', ext)
target_path = dirpath.join(rel_path)
if target_path.exists():
exists = True
# only check anchors in html for now
if ext == ".html" and anchor and self.parent.check_anchors:
with target_path.open() as fpt:
parsed = html5lib.parse(
fpt, namespaceHTMLElements=False)
return self.handle_anchor(parsed, anchor)
break
if not exists:
target_path = dirpath.join(url_path)
raise BrokenLinkError(url, "No such file: %s" % target_path)
def handle(self, connection, headers_only=False):
with self.lock:
if headers_only:
connection.send_response(200)
connection.send_header('Content-Type', 'application/x-mpegurl')
connection.send_header('Connection', 'close')
connection.end_headers()
return
params = { k:[v] for k,v in (unquote(x).split('=') for x in [s2 for s1 in connection.query.split('&') for s2 in s1.split(';')] if '=' in x) }
fmt = params['fmt'][0] if 'fmt' in params else None
exported = self.createPlaylist(connection.headers['Host'], connection.reqtype, fmt).encode('utf-8')
connection.send_response(200)
connection.send_header('Content-Type', 'audio/mpegurl; charset=utf-8')
connection.send_header('Content-Length', str(len(exported)))
connection.send_header('Connection', 'close')
connection.end_headers()
connection.wfile.write(exported)
def gen_str_to_sign(self, req):
"""Generate string to sign using giving prepared request"""
url = urlsplit(req.url)
bucket_name = url.netloc.split(".", 1)[0]
logger.debug(req.headers.items())
ucloud_headers = [(k, v.strip())
for k, v in sorted(req.headers.lower_items())
if k.startswith("x-ucloud-")]
canonicalized_headers = "\n".join(
["{0}:{1}".format(k, v) for k, v in ucloud_headers])
canonicalized_resource = "/{0}{1}".format(bucket_name,
unquote(url.path))
str_to_sign = "\n".join([
req.method,
req.headers.get("content-md5", ""),
req.headers.get("content-type", ""),
req.headers.get("date", self._expires),
canonicalized_headers + canonicalized_resource
])
return str_to_sign
def _update_library(self, host=None, showName=None): # pylint: disable=too-many-locals, too-many-return-statements
"""Handles updating KODI host via HTTP API
Attempts to update the KODI video library for a specific tv show if passed,
otherwise update the whole library if enabled.
Args:
host: KODI webserver host:port
showName: Name of a TV show to specifically target the library update for
Returns:
Returns True or False
"""
if not host:
logger.log(u'No KODI host passed, aborting update', logger.WARNING)
return False
logger.log(u"Updating KODI library via HTTP method for host: " + host, logger.DEBUG)
# if we're doing per-show
if showName:
logger.log(u"Updating library in KODI via HTTP method for show " + showName, logger.DEBUG)
pathSql = 'select path.strPath from path, tvshow, tvshowlinkpath where ' \
'tvshow.c00 = "%s" and tvshowlinkpath.idShow = tvshow.idShow ' \
'and tvshowlinkpath.idPath = path.idPath' % showName
# use this to get xml back for the path lookups
xmlCommand = {
'command': 'SetResponseFormat(webheader;false;webfooter;false;header;<xml>;footer;</xml>;opentag;<tag>;closetag;</tag>;closefinaltag;false)'}
# sql used to grab path(s)
sqlCommand = {'command': 'QueryVideoDatabase(%s)' % pathSql}
# set output back to default
resetCommand = {'command': 'SetResponseFormat()'}
# set xml response format, if this fails then don't bother with the rest
request = self._send_to_kodi(xmlCommand, host)
if not request:
return False
sqlXML = self._send_to_kodi(sqlCommand, host)
request = self._send_to_kodi(resetCommand, host)
if not sqlXML:
logger.log(u"Invalid response for " + showName + " on " + host, logger.DEBUG)
return False
encSqlXML = quote(sqlXML, ':\\/<>')
try:
et = etree.fromstring(encSqlXML)
except SyntaxError as e:
logger.log(u"Unable to parse XML returned from KODI: " + ex(e), logger.ERROR)
return False
paths = et.findall('.//field')
if not paths:
logger.log(u"No valid paths found for " + showName + " on " + host, logger.DEBUG)
return False
for path in paths:
# we do not need it double-encoded, gawd this is dumb
unEncPath = unquote(path.text).decode(app.SYS_ENCODING)
logger.log(u"KODI Updating " + showName + " on " + host + " at " + unEncPath, logger.DEBUG)
updateCommand = {'command': 'ExecBuiltIn', 'parameter': 'KODI.updatelibrary(video, %s)' % unEncPath}
request = self._send_to_kodi(updateCommand, host)
if not request:
logger.log(u"Update of show directory failed on " + showName + " on " + host + " at " + unEncPath, logger.WARNING)
return False
# sleep for a few seconds just to be sure kodi has a chance to finish each directory
if len(paths) > 1:
time.sleep(5)
# do a full update if requested
else:
logger.log(u"Doing Full Library KODI update on host: " + host, logger.DEBUG)
updateCommand = {'command': 'ExecBuiltIn', 'parameter': 'KODI.updatelibrary(video)'}
request = self._send_to_kodi(updateCommand, host)
if not request:
logger.log(u"KODI Full Library update failed on: " + host, logger.WARNING)
return False
return True
def connect(self):
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(self.timeout)
socket_path = unquote(urlparse(self.unix_socket_url).netloc)
sock.connect(socket_path)
self.sock = sock
def get_graph_from_request(api):
"""Process the GET request returning the filtered graph
:param DictionaryService api: The dictionary service
:return: graph: A BEL graph
:rtype: pybel.BELGraph
"""
network_id = request.args.get(GRAPH_ID)
if network_id is not None:
network_id = int(network_id)
if network_id == 0:
network_id = None
seed_method = request.args.get(SEED_TYPE)
if seed_method and seed_method not in SEED_TYPES:
raise ValueError('Invalid seed method: {}'.format(seed_method))
if seed_method and seed_method == SEED_TYPE_PROVENANCE:
seed_data = {}
authors = request.args.getlist(SEED_DATA_AUTHORS)
if authors:
seed_data['authors'] = [unquote(author) for author in authors]
pmids = request.args.getlist(SEED_DATA_PMIDS)
if pmids:
seed_data['pmids'] = pmids
elif seed_method:
seed_data = request.args.getlist(SEED_DATA_NODES)
seed_data = [api.decode_node(h) for h in seed_data]
else:
seed_data = None
expand_nodes = request.args.get(APPEND_PARAM)
remove_nodes = request.args.get(REMOVE_PARAM)
filters = request.args.getlist(FILTERS)
filter_pathologies = request.args.get(FILTER_PATHOLOGIES)
if filter_pathologies and filter_pathologies in {'True', True}:
filter_pathologies = True
if expand_nodes:
expand_nodes = [api.decode_node(h) for h in expand_nodes.split(',')]
if remove_nodes:
remove_nodes = [api.decode_node(h) for h in remove_nodes.split(',')]
annotations = {
k: request.args.getlist(k)
for k in request.args if k not in BLACK_LIST
}
graph = api.query(network_id=network_id,
seed_method=seed_method,
seed_data=seed_data,
expand_nodes=expand_nodes,
remove_nodes=remove_nodes,
filters=filters,
filter_pathologies=filter_pathologies,
**annotations)
return graph
def connect(self):
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(self.timeout)
socket_path = unquote(urlparse(self.unix_socket_url).netloc)
sock.connect(socket_path)
self.sock = sock
def handle(self, connection, headers_only=False):
P2pproxy.logger.debug('Handling request')
hostport = connection.headers['Host']
self.params = {
k: [v]
for k, v in (unquote(x).split('=') for x in [
s2 for s1 in connection.query.split('&')
for s2 in s1.split(';')
] if '=' in x)
}
# /channels/ branch
if connection.reqtype in ('channels', 'channels.m3u'):
if connection.path.endswith('play'): # /channels/play?id=[id]
channel_id = self.get_param('id')
if channel_id is None:
# /channels/play?id=&_=[epoch timestamp] is Torrent-TV widget proxy check
# P2pProxy simply closes connection on this request sending Server header, so do we
if self.get_param('_'):
P2pproxy.logger.debug('Status check')
connection.send_response(200)
connection.send_header('Access-Control-Allow-Origin',
'*')
connection.send_header('Connection', 'close')
connection.send_header('Content-Type',
'text/plain;charset=utf-8')
connection.send_header(
'Server', 'P2pProxy/1.0.4.4 HTTPAceProxy')
connection.wfile.write('\r\n')
return
else:
connection.dieWithError(400,
'Bad request') # Bad request
return
if headers_only:
connection.send_response(200)
connection.send_header('Content-Type', 'video/mpeg')
connection.end_headers()
return
stream_type, stream, translations_list = self.api.stream_source(
channel_id)
name = logo = ''
for channel in translations_list:
if channel.getAttribute('id') == channel_id:
name = channel.getAttribute('name')
logo = channel.getAttribute('logo')
if logo != '' and config.fullpathlogo:
logo = P2pproxy.TTVU + logo
break
if stream_type not in ('torrent', 'contentid'):
connection.dieWithError(
404, 'Unknown stream type: %s' % stream_type,
logging.ERROR)
return
elif stream_type == 'torrent':
connection.path = '/url/%s/stream.mp4' % quote(stream, '')
elif stream_type == 'contentid':
connection.path = '/content_id/%s/stream.mp4' % stream
connection.splittedpath = connection.path.split('/')
connection.reqtype = connection.splittedpath[1].lower()
connection.handleRequest(headers_only,
name,
logo,
fmt=self.get_param('fmt'))
# /channels/?filter=[filter]&group=[group]&type=m3u
elif connection.reqtype == 'channels.m3u' or self.get_param(
'type') == 'm3u':
if headers_only:
connection.send_response(200)
connection.send_header('Content-Type',
'application/x-mpegurl')
connection.end_headers()
return
param_group = self.params.get('group')
param_filter = self.get_param('filter')
if param_filter is None: param_filter = 'all' # default filter
if param_group and 'all' in param_group[0]: param_group = None
translations_list = self.api.translations(param_filter)
playlistgen = PlaylistGenerator(
m3uchanneltemplate=config.m3uchanneltemplate)
P2pproxy.logger.debug('Generating requested m3u playlist')
for channel in translations_list:
group_id = channel.getAttribute('group')
if param_group and not group_id in param_group[0]:
continue # filter channels by &group=1,2,5...
name = channel.getAttribute('name')
group = TorrentTvApi.CATEGORIES[int(group_id)].decode(
'UTF-8')
cid = channel.getAttribute('id')
logo = channel.getAttribute('logo')
if logo != '' and config.fullpathlogo:
logo = P2pproxy.TTVU + logo
fields = {
'name': name,
'id': cid,
'url': cid,
'group': group,
'logo': logo
}
if channel.getAttribute('epg_id') != '0':
fields['tvgid'] = config.tvgid % fields
playlistgen.addItem(fields)
P2pproxy.logger.debug('Exporting m3u playlist')
exported = playlistgen.exportm3u(
hostport=hostport,
header=config.m3uheadertemplate,
fmt=self.get_param('fmt')).encode('utf-8')
connection.send_response(200)
connection.send_header('Content-Type', 'application/x-mpegurl')
connection.send_header('Content-Length', str(len(exported)))
connection.end_headers()
connection.wfile.write(exported)
# /channels/?filter=[filter]
else:
if headers_only:
connection.send_response(200)
connection.send_header('Access-Control-Allow-Origin', '*')
connection.send_header('Connection', 'close')
connection.send_header('Content-Type',
'text/xml;charset=utf-8')
connection.end_headers()
return
param_filter = self.get_param('filter')
if param_filter is None: param_filter = 'all' # default filter
translations_list = self.api.translations(param_filter, True)
P2pproxy.logger.debug('Exporting m3u playlist')
connection.send_response(200)
connection.send_header('Access-Control-Allow-Origin', '*')
connection.send_header('Connection', 'close')
connection.send_header('Content-Type',
'text/xml;charset=utf-8')
connection.send_header('Content-Length',
str(len(translations_list)))
connection.end_headers()
connection.wfile.write(translations_list)
# same as /channels request
elif connection.reqtype == 'xbmc.pvr' and connection.path.endswith(
'playlist'):
connection.send_response(200)
connection.send_header('Access-Control-Allow-Origin', '*')
connection.send_header('Connection', 'close')
connection.send_header('Content-Type', 'text/xml;charset=utf-8')
if headers_only:
connection.end_headers()
return
translations_list = self.api.translations('all', True)
connection.send_header('Content-Length',
str(len(translations_list)))
connection.end_headers()
P2pproxy.logger.debug('Exporting m3u playlist')
connection.wfile.write(translations_list)
# /archive/ branch
elif connection.reqtype == 'archive':
if connection.path.endswith(
('dates', 'dates.m3u')): # /archive/dates.m3u
d = datetime.now()
delta = timedelta(days=1)
playlistgen = PlaylistGenerator()
hostport = connection.headers['Host']
days = int(
self.get_param('days')) if 'days' in self.params else 7
suffix = '&suffix=' + self.get_param(
'suffix') if 'suffix' in self.params else ''
for i in range(days):
dfmt = d.strftime('%d-%m-%Y')
url = 'http://%s/archive/playlist/?date=%s%s' % (
hostport, dfmt, suffix)
playlistgen.addItem({
'group': '',
'tvg': '',
'name': dfmt,
'url': url
})
d -= delta
exported = playlistgen.exportm3u(
hostport,
empty_header=True,
process_url=False,
fmt=self.get_param('fmt')).encode('utf-8')
connection.send_response(200)
connection.send_header('Content-Type', 'application/x-mpegurl')
connection.send_header('Content-Length', str(len(exported)))
connection.end_headers()
connection.wfile.write(exported)
return
elif connection.path.endswith(
('playlist', 'playlist.m3u')): # /archive/playlist.m3u
dates = list()
if 'date' in self.params:
for d in self.params['date']:
dates.append(self.parse_date(d).strftime('%d-%m-%Y'))
else:
d = datetime.now()
delta = timedelta(days=1)
days = int(
self.get_param('days')) if 'days' in self.params else 7
for i in range(days):
dates.append(d.strftime('%d-%m-%Y'))
d -= delta
connection.send_response(200)
connection.send_header('Content-Type', 'application/x-mpegurl')
if headers_only:
connection.end_headers()
return
channels_list = self.api.archive_channels()
hostport = connection.headers['Host']
playlistgen = PlaylistGenerator()
suffix = '&suffix=' + self.get_param(
'suffix') if 'suffix' in self.params else ''
for channel in channels_list:
epg_id = channel.getAttribute('epg_id')
name = channel.getAttribute('name')
logo = channel.getAttribute('logo')
if logo != '' and config.fullpathlogo:
logo = P2pproxy.TTVU + logo
for d in dates:
n = name + ' (' + d + ')' if len(dates) > 1 else name
url = 'http://%s/archive/?type=m3u&date=%s&channel_id=%s%s' % (
hostport, d, epg_id, suffix)
playlistgen.addItem({
'group': name,
'tvg': '',
'name': n,
'url': url,
'logo': logo
})
exported = playlistgen.exportm3u(
hostport,
empty_header=True,
process_url=False,
fmt=self.get_param('fmt')).encode('utf-8')
connection.send_header('Content-Length', str(len(exported)))
connection.end_headers()
connection.wfile.write(exported)
return
elif connection.path.endswith('channels'): # /archive/channels
connection.send_response(200)
connection.send_header('Access-Control-Allow-Origin', '*')
connection.send_header('Connection', 'close')
connection.send_header('Content-Type',
'text/xml;charset=utf-8')
if headers_only: connection.end_headers()
else:
archive_channels = self.api.archive_channels(True)
P2pproxy.logger.debug('Exporting m3u playlist')
connection.send_header('Content-Length',
str(len(archive_channels)))
connection.end_headers()
connection.wfile.write(archive_channels)
return
if connection.path.endswith(
'play'): # /archive/play?id=[record_id]
record_id = self.get_param('id')
if record_id is None:
connection.dieWithError(400, 'Bad request') # Bad request
return
if headers_only:
connection.send_response(200)
connection.send_header("Content-Type", "video/mpeg")
connection.end_headers()
return
stream_type, stream = self.api.archive_stream_source(record_id)
if stream_type not in ('torrent', 'contentid'):
connection.dieWithError(
404, 'Unknown stream type: %s' % stream_type,
logging.ERROR)
return
elif stream_type == 'torrent':
connection.path = '/url/%s/stream.mp4' % quote(stream, '')
elif stream_type == 'contentid':
connection.path = '/content_id/%s/stream.mp4' % stream
connection.splittedpath = connection.path.split('/')
connection.reqtype = connection.splittedpath[1].lower()
connection.handleRequest(headers_only,
fmt=self.get_param('fmt'))
# /archive/?type=m3u&date=[param_date]&channel_id=[param_channel]
elif self.get_param('type') == 'm3u':
if headers_only:
connection.send_response(200)
connection.send_header('Content-Type',
'application/x-mpegurl')
connection.end_headers()
return
playlistgen = PlaylistGenerator()
param_channel = self.get_param('channel_id')
d = self.get_date_param()
if param_channel == '' or param_channel is None:
channels_list = self.api.archive_channels()
for channel in channels_list:
channel_id = channel.getAttribute('epg_id')
try:
records_list = self.api.records(channel_id, d)
channel_name = channel.getAttribute('name')
logo = channel.getAttribute('logo')
if logo != '' and config.fullpathlogo:
logo = P2pproxy.TTVU + logo
for record in records_list:
name = record.getAttribute('name')
record_id = record.getAttribute('record_id')
playlistgen.addItem({
'group': channel_name,
'tvg': '',
'name': name,
'url': record_id,
'logo': logo
})
except:
P2pproxy.logger.debug(
'Failed to load archive for %s' % channel_id)
else:
records_list = self.api.records(param_channel, d)
channels_list = self.api.archive_channels()
P2pproxy.logger.debug('Generating archive m3u playlist')
for record in records_list:
record_id = record.getAttribute('record_id')
channel_id = record.getAttribute('epg_id')
name = record.getAttribute('name')
d = datetime.fromtimestamp(
float(
record.getAttribute('time'))).strftime('%H:%M')
n = '%s %s' % (d, name)
logo = ''
for channel in channels_list:
if channel.getAttribute('epg_id') == channel_id:
channel_name = channel.getAttribute('name')
logo = channel.getAttribute('logo')
if channel_name != '':
name = '(' + channel_name + ') ' + name
if logo != '' and config.fullpathlogo:
logo = P2pproxy.TTVU + logo
playlistgen.addItem({
'group': channel_name,
'name': n,
'url': record_id,
'logo': logo,
'tvg': ''
})
P2pproxy.logger.debug('Exporting m3u playlist')
exported = playlistgen.exportm3u(
hostport,
empty_header=True,
archive=True,
fmt=self.get_param('fmt')).encode('utf-8')
connection.send_response(200)
connection.send_header('Content-Type', 'application/x-mpegurl')
connection.send_header('Content-Length', str(len(exported)))
connection.end_headers()
connection.wfile.write(exported)
# /archive/?date=[param_date]&channel_id=[param_channel]
else:
param_date = self.get_param('date')
if param_date is None: d = datetime.now()
else:
try:
d = parse_date(param_date)
except:
return
param_channel = self.get_param('channel_id')
if param_channel == '' or param_channel is None:
connection.dieWithError(
500,
'Got /archive/ request but no channel_id specified!',
logging.ERROR)
return
connection.send_response(200)
connection.send_header('Access-Control-Allow-Origin', '*')
connection.send_header('Connection', 'close')
connection.send_header('Content-Type',
'text/xml;charset=utf-8')
if headers_only: connection.end_headers()
else:
records_list = self.api.records(param_channel,
d.strftime('%d-%m-%Y'),
True)
P2pproxy.logger.debug('Exporting m3u playlist')
connection.send_header('Content-Length',
str(len(records_list)))
connection.end_headers()
connection.wfile.write(records_list)
# Used to generate logomap for the torrenttv plugin
elif connection.reqtype == 'logos':
translations_list = self.api.translations('all')
last = translations_list[-1]
connection.send_response(200)
connection.send_header('Content-Type', 'text/plain;charset=utf-8')
connection.end_headers()
connection.wfile.write("logobase = '" + P2pproxy.TTVU + "'\n")
connection.wfile.write("logomap = {\n")
for channel in translations_list:
name = channel.getAttribute('name').encode('utf-8')
logo = channel.getAttribute('logo').encode('utf-8')
connection.wfile.write(" u'%s': logobase + '%s'" %
(name, logo))
if not channel == last: connection.wfile.write(",\n")
else: connection.wfile.write("\n")
connection.wfile.write("}\n")
def submit_grade(session, attempt_id, is_group_assignment,
grade, text, filenames, rubrics):
assert isinstance(session, BlackboardSession)
if is_group_assignment:
url = ('https://bb.au.dk/webapps/assignment/' +
'gradeAssignmentRedirector' +
'?course_id=%s' % session.course_id +
'&groupAttemptId=%s' % attempt_id)
else:
url = ('https://bb.au.dk/webapps/assignment/' +
'gradeAssignmentRedirector' +
'?course_id=%s' % session.course_id +
'&attempt_id=%s' % attempt_id)
# We need to fetch the page to get the nonce
response = session.get(url)
document = html5lib.parse(response.content, encoding=response.encoding)
form = document.find('.//h:form[@id="currentAttempt_form"]', NS)
if form is None:
raise ParserError("No <form id=currentAttempt_form>", response)
fields = (form.findall('.//h:input', NS) +
form.findall('.//h:textarea', NS))
data = [
(field.get('name'), form_field_value(field))
for field in fields
if field.get('name')
]
data_lookup = {k: i for i, (k, v) in enumerate(data)}
def data_get(k, *args):
if args:
d, = args
try:
return data[data_lookup[k]][1]
except KeyError:
if args:
return d
raise
def data_set(k, v):
try:
data[data_lookup[k]] = k, v
except KeyError:
data_lookup[k] = len(data)
data.append((k, v))
def data_extend(kvs):
for k, v in kvs:
data_lookup[k] = len(data)
data.append((k, v))
data_set('grade', str(grade))
data_set('feedbacktext', text)
data_set('gradingNotestext',
'Submitted with https://github.com/Mortal/bbfetch')
if rubrics:
rubric_input = '%s_rubricEvaluation' % attempt_id
rubric_data_str = data_get(rubric_input)
rubric_data = json.loads(unquote(rubric_data_str))
for rubric_cells, rubric in zip(rubrics, rubric_data['rubrics']):
rubric['client_changed'] = True
for input_row, row in zip(rubric_cells, rubric['rows']):
row['cell_id'] = input_row
rubric_data_str = quote(json.dumps(rubric_data))
data_set(rubric_input, rubric_data_str)
files = []
for i, filename in enumerate(filenames):
base = os.path.basename(filename)
data_extend([
('feedbackFiles_attachmentType', 'L'),
('feedbackFiles_fileId', 'new'),
('feedbackFiles_artifactFileId', 'undefined'),
('feedbackFiles_artifactType', 'undefined'),
('feedbackFiles_artifactTypeResourceKey', 'undefined'),
('feedbackFiles_linkTitle', base),
])
with open(filename, 'rb') as fp:
fdata = fp.read()
files.append(('feedbackFiles_LocalFile%d' % i, (base, fdata)))
if is_group_assignment:
post_url = (
'https://bb.au.dk/webapps/assignment//gradeGroupAssignment/submit')
else:
post_url = (
'https://bb.au.dk/webapps/assignment//gradeAssignment/submit')
if not files:
# Blackboard requires the POST to be
# Content-Type: multipart/form-data.
# Unfortunately, requests can only make a form-data POST
# if it has file-like input in the files list.
files = [('dummy', io.StringIO(''))]
try:
response = session.post(post_url, data=data, files=files)
except:
logger.exception("data=%r files=%r", data, files)
raise
document = html5lib.parse(response.content, encoding=response.encoding)
badmsg = document.find('.//h:span[@id="badMsg1"]', NS)
if badmsg is not None:
raise ParserError(
"badMsg1: %s" % element_text_content(badmsg), response,
'Post data:\n%s' % pprint.pformat(data),
'Files:\n%s' % pprint.pformat(files))
msg = document.find('.//h:span[@id="goodMsg1"]', NS)
if msg is None:
raise ParserError(
"No goodMsg1 in POST response", response,
'Post data:\n%s' % pprint.pformat(data),
'Files:\n%s' % pprint.pformat(files))
logger.debug("goodMsg1: %s", element_text_content(msg))
def fetch_attempt(session, attempt_id, is_group_assignment):
assert isinstance(session, BlackboardSession)
if is_group_assignment:
url = ('https://bb.au.dk/webapps/assignment/' +
'gradeAssignmentRedirector' +
'?course_id=%s' % session.course_id +
'&groupAttemptId=%s' % attempt_id)
else:
url = ('https://bb.au.dk/webapps/assignment/' +
'gradeAssignmentRedirector' +
'?course_id=%s' % session.course_id +
'&attempt_id=%s' % attempt_id)
l = blackboard.slowlog()
response = session.get(url)
l("Fetching attempt took %.1f s")
document = html5lib.parse(response.content, encoding=response.encoding)
submission_text = document.find(
'.//h:div[@id="submissionTextView"]', NS)
if submission_text is not None:
submission_text = element_to_markdown(submission_text)
comments = document.find(
'.//h:div[@id="currentAttempt_comments"]', NS)
if comments is not None:
xpath = './/h:div[@class="vtbegenerated"]'
comments = [
element_to_markdown(e)
for e in comments.findall(xpath, NS)
]
if not comments:
raise blackboard.ParserError(
"Page contains currentAttempt_comments, " +
"but it contains no comments",
response)
comments = '\n\n'.join(comments)
files = []
submission_list = document.find(
'.//h:ul[@id="currentAttempt_submissionList"]', NS)
if submission_list is None:
raise ParserError("No currentAttempt_submissionList",
response)
for submission in submission_list:
filename = element_text_content(submission)
download_button = submission.find(
'.//h:a[@class="dwnldBtn"]', NS)
if download_button is not None:
download_link = urljoin(
response.url, download_button.get('href'))
files.append(
dict(filename=filename, download_link=download_link))
else:
s = 'currentAttempt_attemptFilesubmissionText'
a = submission.find(
'.//h:a[@id="' + s + '"]', NS)
if a is not None:
# This <li> is for the submission_text
if not submission_text:
raise blackboard.ParserError(
"%r in file list, but no " % (filename,) +
"accompanying submission text contents",
response)
else:
raise blackboard.ParserError(
"No download link for file %r" % (filename,),
response)
score_input = document.find(
'.//h:input[@id="currentAttempt_grade"]', NS)
if score_input is None:
score = None
else:
score = form_field_value(score_input)
try:
score = float(score)
except ValueError:
if score:
raise blackboard.ParserError(
"Couldn't parse currentAttempt_grade: %r" % (score,),
response)
score = None
feedbacktext_input = document.find(
'.//*[@id="feedbacktext"]', NS)
if feedbacktext_input is None:
feedback = ''
else:
feedback = form_field_value(feedbacktext_input)
if '<' in feedback:
feedback = html_to_markdown(feedback)
gradingNotestext_input = document.find(
'.//*[@id="gradingNotestext"]', NS)
if gradingNotestext_input is None:
grading_notes = ''
else:
grading_notes = form_field_value(gradingNotestext_input)
feedbackfiles_rows = document.find(
'.//h:tbody[@id="feedbackFiles_table_body"]', NS)
feedbackfiles = []
for i, row in enumerate(feedbackfiles_rows or []):
try:
link = row.findall('.//h:a', NS)[0]
except IndexError:
raise blackboard.ParserError(
"feedbackFiles_table_body row %s: no link" % i,
response)
download_link = urljoin(
response.url, link.get('href'))
filename = element_text_content(link)
feedbackfiles.append(
dict(filename=filename, download_link=download_link))
rubric_data = None
if is_group_assignment:
rubric_input = document.find(
'.//h:input[@id="%s_rubricEvaluation"]' % attempt_id, NS)
if rubric_input is not None:
rubric_data_str = form_field_value(rubric_input)
try:
rubric_data = json.loads(unquote(rubric_data_str))
except JSONDecodeError:
raise ParserError("Couldn't decode JSON", response)
t1 = 'blackboard.platform.gradebook2.GroupAttempt'
t2 = 'blackboard.plugin.rubric.api.core.data.EvaluationEntity'
if rubric_data['evalDataType'] == t1:
if rubric_data['evalEntityId'] != attempt_id:
raise ParserError(
"evalEntityId is %r, expected %r" %
(rubric_data['evalEntityId'], attempt_id),
response)
elif rubric_data['evalDataType'] == t2:
# Seems to indicate an already filled-out rubric
pass
else:
raise ParserError(
"Unknown evalDataType %r" % rubric_data['evalDataType'],
response)
return dict(
submission=submission_text,
comments=comments,
files=files,
feedback=feedback,
feedbackfiles=feedbackfiles,
score=score,
grading_notes=grading_notes,
rubric_data=rubric_data,
)
def _update_library(self, host=None, showName=None): # pylint: disable=too-many-locals, too-many-return-statements
"""Handles updating KODI host via HTTP API
Attempts to update the KODI video library for a specific tv show if passed,
otherwise update the whole library if enabled.
Args:
host: KODI webserver host:port
showName: Name of a TV show to specifically target the library update for
Returns:
Returns True or False
"""
if not host:
logger.log(u'No KODI host passed, aborting update', logger.WARNING)
return False
logger.log(u"Updating KODI library via HTTP method for host: " + host, logger.DEBUG)
# if we're doing per-show
if showName:
logger.log(u"Updating library in KODI via HTTP method for show " + showName, logger.DEBUG)
pathSql = 'select path.strPath from path, tvshow, tvshowlinkpath where ' \
'tvshow.c00 = "%s" and tvshowlinkpath.idShow = tvshow.idShow ' \
'and tvshowlinkpath.idPath = path.idPath' % showName
# use this to get xml back for the path lookups
xmlCommand = {
'command': 'SetResponseFormat(webheader;false;webfooter;false;header;<xml>;footer;</xml>;opentag;<tag>;closetag;</tag>;closefinaltag;false)'}
# sql used to grab path(s)
sqlCommand = {'command': 'QueryVideoDatabase(%s)' % pathSql}
# set output back to default
resetCommand = {'command': 'SetResponseFormat()'}
# set xml response format, if this fails then don't bother with the rest
request = self._send_to_kodi(xmlCommand, host)
if not request:
return False
sqlXML = self._send_to_kodi(sqlCommand, host)
request = self._send_to_kodi(resetCommand, host)
if not sqlXML:
logger.log(u"Invalid response for " + showName + " on " + host, logger.DEBUG)
return False
encSqlXML = quote(sqlXML, ':\\/<>')
try:
et = etree.fromstring(encSqlXML)
except SyntaxError as e:
logger.log(u"Unable to parse XML returned from KODI: " + ex(e), logger.ERROR)
return False
paths = et.findall('.//field')
if not paths:
logger.log(u"No valid paths found for " + showName + " on " + host, logger.DEBUG)
return False
for path in paths:
# we do not need it double-encoded, gawd this is dumb
unEncPath = unquote(path.text).decode(sickbeard.SYS_ENCODING)
logger.log(u"KODI Updating " + showName + " on " + host + " at " + unEncPath, logger.DEBUG)
updateCommand = {'command': 'ExecBuiltIn', 'parameter': 'KODI.updatelibrary(video, %s)' % unEncPath}
request = self._send_to_kodi(updateCommand, host)
if not request:
logger.log(u"Update of show directory failed on " + showName + " on " + host + " at " + unEncPath, logger.WARNING)
return False
# sleep for a few seconds just to be sure kodi has a chance to finish each directory
if len(paths) > 1:
time.sleep(5)
# do a full update if requested
else:
logger.log(u"Doing Full Library KODI update on host: " + host, logger.DEBUG)
updateCommand = {'command': 'ExecBuiltIn', 'parameter': 'KODI.updatelibrary(video)'}
request = self._send_to_kodi(updateCommand, host)
if not request:
logger.log(u"KODI Full Library update failed on: " + host, logger.WARNING)
return False
return True
def fetch_attempt(session, attempt_id, is_group_assignment):
assert isinstance(session, BlackboardSession)
if is_group_assignment:
url = ('https://%s/webapps/assignment/' % DOMAIN +
'gradeAssignmentRedirector' +
'?course_id=%s' % session.course_id +
'&groupAttemptId=%s' % attempt_id)
else:
url = ('https://%s/webapps/assignment/' % DOMAIN +
'gradeAssignmentRedirector' +
'?course_id=%s' % session.course_id +
'&attempt_id=%s' % attempt_id)
l = blackboard.slowlog()
response = session.get(url)
l("Fetching attempt took %.1f s")
document = html5lib.parse(response.content,
transport_encoding=response.encoding)
currentAttempt_container = document.find('.//h:div[@id="currentAttempt"]',
NS)
if currentAttempt_container is None:
not_yet_submitted = ('This attempt has not yet been submitted and ' +
'is not available to view at present.')
if not_yet_submitted in response.text:
raise NotYetSubmitted
raise blackboard.ParserError('No <div id="currentAttempt">',
response=response)
submission_text = document.find('.//h:div[@id="submissionTextView"]', NS)
if submission_text is not None:
submission_text = element_to_markdown(submission_text)
comments = document.find('.//h:div[@id="currentAttempt_comments"]', NS)
if comments is not None:
xpath = './/h:div[@class="vtbegenerated"]'
comments = [
element_to_markdown(e) for e in comments.findall(xpath, NS)
]
if not comments:
raise blackboard.ParserError(
"Page contains currentAttempt_comments, " +
"but it contains no comments", response)
comments = '\n\n'.join(comments)
files = []
submission_list = document.find(
'.//h:ul[@id="currentAttempt_submissionList"]', NS)
if submission_list is None:
if comments is None and submission_text is None:
logger.warning("The submission is completely empty.")
elif submission_text is None:
logger.warning(
"No submission; the student only uploaded a comment.")
else:
logger.warning("The student only uploaded a text submission.")
submission_list = ()
for submission in submission_list:
filename = element_text_content(submission)
download_button = submission.find('.//h:a[@class="dwnldBtn"]', NS)
if download_button is not None:
download_link = urljoin(response.url, download_button.get('href'))
files.append(dict(filename=filename, download_link=download_link))
else:
s = 'currentAttempt_attemptFilesubmissionText'
a = submission.find('.//h:a[@id="' + s + '"]', NS)
if a is not None:
# This <li> is for the submission_text
if not submission_text:
raise blackboard.ParserError(
"%r in file list, but no " % (filename, ) +
"accompanying submission text contents", response)
else:
raise blackboard.ParserError(
"No download link for file %r" % (filename, ), response)
score_input = document.find('.//h:input[@id="currentAttempt_grade"]', NS)
if score_input is None:
score = None
else:
score = form_field_value(score_input)
try:
score = float(score)
except ValueError:
if score:
raise blackboard.ParserError(
"Couldn't parse currentAttempt_grade: %r" % (score, ),
response)
score = None
feedbacktext_input = document.find('.//*[@id="feedbacktext"]', NS)
if feedbacktext_input is None:
feedback = ''
else:
feedback = form_field_value(feedbacktext_input)
if '<' in feedback:
feedback = html_to_markdown(feedback)
gradingNotestext_input = document.find('.//*[@id="gradingNotestext"]', NS)
if gradingNotestext_input is None:
grading_notes = ''
else:
grading_notes = form_field_value(gradingNotestext_input)
feedbackfiles_rows = document.find(
'.//h:tbody[@id="feedbackFiles_table_body"]', NS)
feedbackfiles = []
for i, row in enumerate(feedbackfiles_rows or []):
try:
link = row.findall('.//h:a', NS)[0]
except IndexError:
raise blackboard.ParserError(
"feedbackFiles_table_body row %s: no link" % i, response)
download_link = urljoin(response.url, link.get('href'))
filename = element_text_content(link)
feedbackfiles.append(
dict(filename=filename, download_link=download_link))
rubric_data = None
if is_group_assignment:
rubric_input = document.find(
'.//h:input[@id="%s_rubricEvaluation"]' % attempt_id, NS)
if rubric_input is not None:
rubric_data_str = form_field_value(rubric_input)
try:
rubric_data = json.loads(unquote(rubric_data_str))
except JSONDecodeError:
raise ParserError("Couldn't decode JSON", response)
t1 = 'blackboard.platform.gradebook2.GroupAttempt'
t2 = 'blackboard.plugin.rubric.api.core.data.EvaluationEntity'
if rubric_data['evalDataType'] == t1:
if rubric_data['evalEntityId'] != attempt_id:
raise ParserError(
"evalEntityId is %r, expected %r" %
(rubric_data['evalEntityId'], attempt_id), response)
elif rubric_data['evalDataType'] == t2:
# Seems to indicate an already filled-out rubric
pass
else:
raise ParserError(
"Unknown evalDataType %r" % rubric_data['evalDataType'],
response)
return dict(
submission=submission_text,
comments=comments,
files=files,
feedback=feedback,
feedbackfiles=feedbackfiles,
score=score,
grading_notes=grading_notes,
rubric_data=rubric_data,
)
def send(self, request, **kwargs):
""" Wraps a file, described in request, in a Response object.
:param request: The PreparedRequest` being "sent".
:returns: a Response object containing the file
"""
# Check that the method makes sense. Only support GET
if request.method != "GET":
raise ValueError("Invalid request method %s" % request.method)
# Parse the URL
url_parts = urlparse(request.url)
# Reject URLs with a hostname component
if url_parts.netloc and url_parts.netloc != "localhost":
raise ValueError(
"file: URLs with hostname components are not permitted")
resp = Response()
# Open the file, translate certain errors into HTTP responses
# Use urllib's unquote to translate percent escapes into whatever
# they actually need to be
try:
# Split the path on / (the URL directory separator) and decode any
# % escapes in the parts
path_parts = [unquote(p) for p in url_parts.path.split('/')]
# If os.sep is in any of the parts, someone fed us some shenanigans.
# Treat is like a missing file.
if any(os.sep in p for p in path_parts):
raise IOError(errno.ENOENT, os.strerror(errno.ENOENT))
# Put it the path back together
# [0] will be an empty string, so stick os.sep in there to make
# the path absolute
path_parts[0] = os.sep
path = os.path.join(*path_parts)
resp.raw = open(path, "rb")
except IOError as e:
if e.errno == errno.EACCES:
resp.status_code = 403
elif e.errno == errno.ENOENT:
resp.status_code = 404
else:
resp.status_code = 400
# Wrap the error message in a file-like object
# The error message will be localized, try to convert the string
# representation of the exception into a byte stream
resp_str = str(e).encode(locale.nl_langinfo(locale.CODESET))
resp.raw = BytesIO(resp_str)
resp.headers['Content-Length'] = len(resp_str)
else:
resp.status_code = 200
# If it's a regular file, set the Content-Length
resp_stat = os.fstat(resp.raw.fileno())
if stat.S_ISREG(resp_stat.st_mode):
resp.headers['Content-Length'] = resp_stat.st_size
return resp
def send(self, request, **kwargs):
""" Wraps a file, described in request, in a Response object.
:param request: The PreparedRequest` being "sent".
:returns: a Response object containing the file
"""
# Check that the method makes sense. Only support GET
if request.method not in ("GET", "HEAD"):
raise ValueError("Invalid request method %s" % request.method)
# Parse the URL
url_parts = urlparse(request.url)
# Reject URLs with a hostname component
if url_parts.netloc and url_parts.netloc != "localhost":
raise ValueError("file: URLs with hostname components are not permitted")
resp = Response()
# Open the file, translate certain errors into HTTP responses
# Use urllib's unquote to translate percent escapes into whatever
# they actually need to be
try:
# Split the path on / (the URL directory separator) and decode any
# % escapes in the parts
path_parts = [unquote(p) for p in url_parts.path.split('/')]
# Strip out the leading empty parts created from the leading /'s
while path_parts and not path_parts[0]:
path_parts.pop(0)
# If os.sep is in any of the parts, someone fed us some shenanigans.
# Treat is like a missing file.
if any(os.sep in p for p in path_parts):
raise IOError(errno.ENOENT, os.strerror(errno.ENOENT))
# Look for a drive component. If one is present, store it separately
# so that a directory separator can correctly be added to the real
# path, and remove any empty path parts between the drive and the path.
# Assume that a part ending with : or | (legacy) is a drive.
if path_parts and (path_parts[0].endswith('|') or
path_parts[0].endswith(':')):
path_drive = path_parts.pop(0)
if path_drive.endswith('|'):
path_drive = path_drive[:-1] + ':'
while path_parts and not path_parts[0]:
path_parts.pop(0)
else:
path_drive = ''
# Try to put the path back together
# Join the drive back in, and stick os.sep in front of the path to
# make it absolute.
path = path_drive + os.sep + os.path.join(*path_parts)
# Check if the drive assumptions above were correct. If path_drive
# is set, and os.path.splitdrive does not return a drive, it wasn't
# reall a drive. Put the path together again treating path_drive
# as a normal path component.
if path_drive and not os.path.splitdrive(path):
path = os.sep + os.path.join(path_drive, *path_parts)
# Use io.open since we need to add a release_conn method, and
# methods can't be added to file objects in python 2.
resp.raw = io.open(path, "rb")
resp.raw.release_conn = resp.raw.close
except IOError as e:
if e.errno == errno.EACCES:
resp.status_code = codes.forbidden
elif e.errno == errno.ENOENT:
resp.status_code = codes.not_found
else:
resp.status_code = codes.bad_request
# Wrap the error message in a file-like object
# The error message will be localized, try to convert the string
# representation of the exception into a byte stream
resp_str = str(e).encode(locale.getpreferredencoding(False))
resp.raw = BytesIO(resp_str)
resp.headers['Content-Length'] = len(resp_str)
# Add release_conn to the BytesIO object
resp.raw.release_conn = resp.raw.close
else:
resp.status_code = codes.ok
# If it's a regular file, set the Content-Length
resp_stat = os.fstat(resp.raw.fileno())
if stat.S_ISREG(resp_stat.st_mode):
resp.headers['Content-Length'] = resp_stat.st_size
return resp
def get_connection(self, url, proxies=None):
# proxies, silently ignored
path = unquote(urlparse(url).netloc)
return HTTPUnixConnectionPool(path)
def send(self, request, **kwargs):
"""Wraps a file, described in request, in a Response object.
:param request: The PreparedRequest` being "sent".
:returns: a Response object containing the file
"""
# Check that the method makes sense. Only support GET
if request.method not in ("GET", "HEAD"):
raise ValueError("Invalid request method %s" % request.method)
# Parse the URL
url_parts = urlparse(request.url)
# Reject URLs with a hostname component
if url_parts.netloc and url_parts.netloc != "localhost":
raise ValueError(
"file: URLs with hostname components are not permitted")
resp = Response()
# Give the Response some context.
resp.request = request
# Open the file, translate certain errors into HTTP responses
# Use urllib's unquote to translate percent escapes into whatever
# they actually need to be
try:
# Split the path on / (the URL directory separator) and decode any
# % escapes in the parts
path_parts = [unquote(p) for p in url_parts.path.split("/")]
# Strip out the leading empty parts created from the leading /'s
while path_parts and not path_parts[0]:
path_parts.pop(0)
# If os.sep is in any of the parts, someone fed us some shenanigans.
# Treat is like a missing file.
if any(os.sep in p for p in path_parts):
raise IOError(errno.ENOENT, os.strerror(errno.ENOENT))
# Look for a drive component. If one is present, store it separately
# so that a directory separator can correctly be added to the real
# path, and remove any empty path parts between the drive and the path.
# Assume that a part ending with : or | (legacy) is a drive.
if path_parts and (path_parts[0].endswith("|")
or path_parts[0].endswith(":")):
path_drive = path_parts.pop(0)
if path_drive.endswith("|"):
path_drive = path_drive[:-1] + ":"
while path_parts and not path_parts[0]:
path_parts.pop(0)
else:
path_drive = ""
# Try to put the path back together
# Join the drive back in,
# don't stick os.sep in front of the path to keep it relative
path = path_drive + os.path.join(*path_parts)
# Check if the drive assumptions above were correct. If path_drive
# is set, and os.path.splitdrive does not return a drive, it wasn't
# reall a drive. Put the path together again treating path_drive
# as a normal path component.
if path_drive and not os.path.splitdrive(path):
path = os.sep + os.path.join(path_drive, *path_parts)
# Use io.open since we need to add a release_conn method, and
# methods can't be added to file objects in python 2.
resp.raw = io.open(path, "rb")
resp.raw.release_conn = resp.raw.close
except IOError as e:
if e.errno == errno.EACCES:
resp.status_code = codes.forbidden
elif e.errno == errno.ENOENT:
resp.status_code = codes.not_found
else:
resp.status_code = codes.bad_request
# Wrap the error message in a file-like object
# The error message will be localized, try to convert the string
# representation of the exception into a byte stream
resp_str = str(e).encode(locale.getpreferredencoding(False))
resp.raw = io.BytesIO(resp_str)
if self._set_content_length:
resp.headers["Content-Length"] = len(resp_str)
# Add release_conn to the BytesIO object
resp.raw.release_conn = resp.raw.close
else:
resp.status_code = codes.ok
resp.url = request.url
# If it's a regular file, set the Content-Length
resp_stat = os.fstat(resp.raw.fileno())
if stat.S_ISREG(resp_stat.st_mode) and self._set_content_length:
resp.headers["Content-Length"] = resp_stat.st_size
# set Date in RFC 1123 from file mtime
resp.headers["Date"] = formatdate(timeval=resp_stat.st_mtime,
localtime=False,
usegmt=True)
return resp
def send(self, request, **kwargs):
""" Wraps a file, described in request, in a Response object.
:param request: The PreparedRequest` being "sent".
:returns: a Response object containing the file
"""
# Check that the method makes sense. Only support GET
if request.method not in ("GET", "HEAD"):
raise ValueError("Invalid request method %s" % request.method)
# Parse the URL
url_parts = urlparse(request.url)
# Make the Windows URLs slightly nicer
if is_win32 and url_parts.netloc.endswith(":"):
url_parts = url_parts._replace(path="/" + url_parts.netloc +
url_parts.path,
netloc='')
# Reject URLs with a hostname component
if url_parts.netloc and url_parts.netloc not in ("localhost", ".",
"..", "-"):
raise ValueError(
"file: URLs with hostname components are not permitted")
# If the path is relative update it to be absolute
if url_parts.netloc in (".", ".."):
pwd = os.path.abspath(url_parts.netloc).replace(os.sep, "/") + "/"
if is_win32:
# prefix the path with a / in Windows
pwd = "/" + pwd
url_parts = url_parts._replace(
path=urljoin(pwd, url_parts.path.lstrip("/")))
resp = Response()
resp.url = request.url
# Open the file, translate certain errors into HTTP responses
# Use urllib's unquote to translate percent escapes into whatever
# they actually need to be
try:
# If the netloc is - then read from stdin
if url_parts.netloc == "-":
if is_py3:
resp.raw = sys.stdin.buffer
else:
resp.raw = sys.stdin
# make a fake response URL, the current directory
resp.url = "file://" + os.path.abspath(".").replace(
os.sep, "/") + "/"
else:
# Split the path on / (the URL directory separator) and decode any
# % escapes in the parts
path_parts = [unquote(p) for p in url_parts.path.split('/')]
# Strip out the leading empty parts created from the leading /'s
while path_parts and not path_parts[0]:
path_parts.pop(0)
# If os.sep is in any of the parts, someone fed us some shenanigans.
# Treat is like a missing file.
if any(os.sep in p for p in path_parts):
raise IOError(errno.ENOENT, os.strerror(errno.ENOENT))
# Look for a drive component. If one is present, store it separately
# so that a directory separator can correctly be added to the real
# path, and remove any empty path parts between the drive and the path.
# Assume that a part ending with : or | (legacy) is a drive.
if path_parts and (path_parts[0].endswith('|')
or path_parts[0].endswith(':')):
path_drive = path_parts.pop(0)
if path_drive.endswith('|'):
path_drive = path_drive[:-1] + ':'
while path_parts and not path_parts[0]:
path_parts.pop(0)
else:
path_drive = ''
# Try to put the path back together
# Join the drive back in, and stick os.sep in front of the path to
# make it absolute.
path = path_drive + os.sep + os.path.join(*path_parts)
# Check if the drive assumptions above were correct. If path_drive
# is set, and os.path.splitdrive does not return a drive, it wasn't
# reall a drive. Put the path together again treating path_drive
# as a normal path component.
if path_drive and not os.path.splitdrive(path):
path = os.sep + os.path.join(path_drive, *path_parts)
# Use io.open since we need to add a release_conn method, and
# methods can't be added to file objects in python 2.
resp.raw = io.open(path, "rb")
resp.raw.release_conn = resp.raw.close
except IOError as e:
if e.errno == errno.EACCES:
resp.status_code = codes.forbidden
elif e.errno == errno.ENOENT:
resp.status_code = codes.not_found
else:
resp.status_code = codes.bad_request
# Wrap the error message in a file-like object
# The error message will be localized, try to convert the string
# representation of the exception into a byte stream
resp_str = str(e).encode(locale.getpreferredencoding(False))
resp.raw = BytesIO(resp_str)
resp.headers['Content-Length'] = len(resp_str)
# Add release_conn to the BytesIO object
resp.raw.release_conn = resp.raw.close
else:
resp.status_code = codes.ok
# If it's a regular file, set the Content-Length
resp_stat = os.fstat(resp.raw.fileno())
if stat.S_ISREG(resp_stat.st_mode):
resp.headers['Content-Length'] = resp_stat.st_size
return resp
def submit_grade(session, attempt_id, is_group_assignment,
grade, text, filenames, rubrics):
assert isinstance(session, BlackboardSession)
if is_group_assignment:
url = ('https://bb.au.dk/webapps/assignment/' +
'gradeAssignmentRedirector' +
'?course_id=%s' % session.course_id +
'&groupAttemptId=%s' % attempt_id)
else:
url = ('https://bb.au.dk/webapps/assignment/' +
'gradeAssignmentRedirector' +
'?course_id=%s' % session.course_id +
'&attempt_id=%s' % attempt_id)
# We need to fetch the page to get the nonce
response = session.get(url)
document = html5lib.parse(response.content, encoding=response.encoding)
form = document.find('.//h:form[@id="currentAttempt_form"]', NS)
if form is None:
raise ParserError("No <form id=currentAttempt_form>", response)
fields = (form.findall('.//h:input', NS) +
form.findall('.//h:textarea', NS))
data = [
(field.get('name'), form_field_value(field))
for field in fields
if field.get('name')
]
data_lookup = {k: i for i, (k, v) in enumerate(data)}
def data_get(k, *args):
if args:
d, = args
try:
return data[data_lookup[k]][1]
except KeyError:
if args:
return d
raise
def data_set(k, v):
try:
data[data_lookup[k]] = k, v
except KeyError:
data_lookup[k] = len(data)
data.append((k, v))
def data_extend(kvs):
for k, v in kvs:
data_lookup[k] = len(data)
data.append((k, v))
data_set('grade', str(grade))
data_set('feedbacktext', text)
data_set('gradingNotestext',
'Submitted with https://github.com/Mortal/bbfetch')
if rubrics:
rubric_input = '%s_rubricEvaluation' % attempt_id
rubric_data_str = data_get(rubric_input)
rubric_data = json.loads(unquote(rubric_data_str))
for rubric_cells, rubric in zip(rubrics, rubric_data['rubrics']):
rubric['client_changed'] = True
for input_row, row in zip(rubric_cells, rubric['rows']):
row['cell_id'] = input_row
rubric_data_str = quote(json.dumps(rubric_data))
data_set(rubric_input, rubric_data_str)
files = []
for i, filename in enumerate(filenames):
base = os.path.basename(filename)
data_extend([
('feedbackFiles_attachmentType', 'L'),
('feedbackFiles_fileId', 'new'),
('feedbackFiles_artifactFileId', 'undefined'),
('feedbackFiles_artifactType', 'undefined'),
('feedbackFiles_artifactTypeResourceKey', 'undefined'),
('feedbackFiles_linkTitle', base),
])
with open(filename, 'rb') as fp:
fdata = fp.read()
files.append(('feedbackFiles_LocalFile%d' % i, (base, fdata)))
if is_group_assignment:
post_url = (
'https://bb.au.dk/webapps/assignment//gradeGroupAssignment/submit')
else:
post_url = (
'https://bb.au.dk/webapps/assignment//gradeAssignment/submit')
if not files:
# Blackboard requires the POST to be
# Content-Type: multipart/form-data.
# Unfortunately, requests can only make a form-data POST
# if it has file-like input in the files list.
files = [('dummy', io.StringIO(''))]
try:
response = session.post(post_url, data=data, files=files)
except:
logger.exception("data=%r files=%r", data, files)
raise
document = html5lib.parse(response.content, encoding=response.encoding)
badmsg = document.find('.//h:span[@id="badMsg1"]', NS)
if badmsg is not None:
raise ParserError(
"badMsg1: %s" % element_text_content(badmsg), response,
'Post data:\n%s' % pprint.pformat(data),
'Files:\n%s' % pprint.pformat(files))
msg = document.find('.//h:span[@id="goodMsg1"]', NS)
if msg is None:
raise ParserError(
"No goodMsg1 in POST response", response,
'Post data:\n%s' % pprint.pformat(data),
'Files:\n%s' % pprint.pformat(files))
logger.debug("goodMsg1: %s", element_text_content(msg))
def send(self, request, **kwargs):
""" Wraps a file, described in request, in a Response object.
:param request: The PreparedRequest` being "sent".
:returns: a Response object containing the file
"""
# Check that the method makes sense. Only support GET
if request.method != "GET":
raise ValueError("Invalid request method %s" % request.method)
# Parse the URL
url_parts = urlparse(request.url)
# Reject URLs with a hostname component
if url_parts.netloc and url_parts.netloc != "localhost":
raise ValueError("file: URLs with hostname components are not permitted")
resp = Response()
# Open the file, translate certain errors into HTTP responses
# Use urllib's unquote to translate percent escapes into whatever
# they actually need to be
try:
# Split the path on / (the URL directory separator) and decode any
# % escapes in the parts
path_parts = [unquote(p) for p in url_parts.path.split('/')]
# If os.sep is in any of the parts, someone fed us some shenanigans.
# Treat is like a missing file.
if any(os.sep in p for p in path_parts):
raise IOError(errno.ENOENT, os.strerror(errno.ENOENT))
# Put it the path back together
# [0] will be an empty string, so stick os.sep in there to make
# the path absolute
path_parts[0] = os.sep
path = os.path.join(*path_parts)
resp.raw = open(path, "rb")
except IOError as e:
if e.errno == errno.EACCES:
resp.status_code = 403
elif e.errno == errno.ENOENT:
resp.status_code = 404
else:
resp.status_code = 400
# Wrap the error message in a file-like object
# The error message will be localized, try to convert the string
# representation of the exception into a byte stream
resp_str = str(e).encode(locale.nl_langinfo(locale.CODESET))
resp.raw = BytesIO(resp_str)
resp.headers['Content-Length'] = len(resp_str)
else:
resp.status_code = 200
# If it's a regular file, set the Content-Length
resp_stat = os.fstat(resp.raw.fileno())
if stat.S_ISREG(resp_stat.st_mode):
resp.headers['Content-Length'] = resp_stat.st_size
return resp
def unescape(_str):
return unicode(unquote(_str)).encode('utf8')
def get_connection(self, url, proxies=None):
# proxies, silently ignored
path = unquote(urlparse(url).netloc)
return HTTPUnixConnectionPool(path)
def latest_version():
r = requests.get(BASE_URL)
soup = BeautifulSoup(r.text, 'html.parser')
hrefs = (x.get('href') for x in soup.find_all(
href=lambda x: x and not x.startswith('.')))
return unquote(max(hrefs)).strip('/')
