def getLecuresFor(self, url):
src = urlread(url)
div_tags = BS(src, parseOnlyThese=SS('div', {'class': 'medialisting'}))
#attempt to parse normal page
if len(div_tags) > 0:
items = self.parse_normal_course(div_tags)
else:
items = self.parse_rm_course(src)
return items
def extractMediaFile(self, url, preferArchive=False):
ocw_page_url = url
src = urlread(ocw_page_url)
if preferArchive:
archive = self.extractInternetArchiveFiles(src=src)
if archive:
return archive
youtube_url = self.extractYouTubeFile(src=src)
return youtube_url
def _query(query,
title=None,
titles=None,
project=_DEFAULT_PROJECT,
format=formats.JSON,
parse=True):
formats.check(format, _SUPPORTED_FORMATS)
full_query = {
'format': format,
'action': 'query',
'titles': _get_titles(title, titles)
}
full_query.update(query)
response = utils.urlread(_get_url(project), full_query)
return formats.parse(response, format, parse)
def getAllCourse(self, includeSelected=False):
src = urlread(self.courses_url)
div_tags = BS(src, parseOnlyThese=SS('tr', {'class': re.compile('row|alt-row')}))
#filter out classes that don't have full video lectures available
if includeSelected:
lectureFilter = re.compile('Video lectures|Selected video lectures')
else:
lectureFilter = 'Video lectures'
video_divs = filter(lambda d: d.find('a', {'title': lectureFilter}), div_tags)
items = [{'name': div.u.string,
'url': self.urljoin(div.a['href']),
'mode': MODE_MIT_LECTURES} for div in video_divs]
return items
def build_initrd(self,initrd,kickstart,data):
"""
Crack open an initrd and install the kickstart file.
"""
# save kickstart to file
ksdata = utils.urlread(kickstart)
fd = open("/var/spool/koan/ks.cfg","w+")
if ksdata is not None:
fd.write(ksdata)
fd.close()
# handle insertion of kickstart based on type of initrd
fd = open("/var/spool/koan/insert.sh","w+")
fd.write(self.get_insert_script(initrd))
fd.close()
utils.subprocess_call([ "/bin/bash", "/var/spool/koan/insert.sh" ])
shutil.copyfile("/var/spool/koan/initrd_final", initrd)
def get_entity(id, version=None, format=formats.JSON, parse=True):
"""Retrieve entity data from WikiData.
See https://www.wikidata.org/wiki/Wikidata:Data_access#Linked_Data_interface
Args:
id: The entity ID (str)
version: The entity version (str or int, optional)
format: The response format (str, optional)
parse: Whether to parse the response (boolean, optional)
Returns:
The response (str or object)
"""
formats.check(format, _GET_ENTITY_FORMATS)
url = _GET_ENTITY_BASE_URL + id + '.' + format
query = {'version': str(version)} if version else None
response = utils.urlread(url, query)
return formats.parse(response, format, parse)
def get_install_tree_from_kickstart(self,profile_data):
"""
Scan the kickstart configuration for either a "url" or "nfs" command
take the install_tree url from that
"""
try:
raw = utils.urlread(profile_data["kickstart"])
lines = raw.splitlines()
method_re = re.compile('(?P<urlcmd>\s*url\s.*)|(?P<nfscmd>\s*nfs\s.*)')
url_parser = opt_parse.OptionParser()
url_parser.add_option("--url", dest="url")
nfs_parser = opt_parse.OptionParser()
nfs_parser.add_option("--dir", dest="dir")
nfs_parser.add_option("--server", dest="server")
for line in lines:
match = method_re.match(line)
if match:
cmd = match.group("urlcmd")
if cmd:
(options,args) = url_parser.parse_args(cmd.split()[1:])
profile_data["install_tree"] = options.url
break
cmd = match.group("nfscmd")
if cmd:
(options,args) = nfs_parser.parse_args(cmd.split()[1:])
profile_data["install_tree"] = "nfs://%s:%s" % (options.server,options.dir)
break
if self.safe_load(profile_data,"install_tree"):
print "install_tree:", profile_data["install_tree"]
else:
print "warning: kickstart found but no install_tree found"
except:
# unstable to download the kickstart, however this might not
# be an error. For instance, xen FV installations of non
# kickstart OS's...
pass
def _get(parameters, format=formats.JSON, parse=True): formats.check(format, _SUPPORTED_FORMATS) response = utils.urlread(_WIKIMEDIA_API_URL + '/'.join(parameters)) return formats.parse(response, format, parse)
def query(query, format=formats.JSON, parse=True):
formats.check(format, _QUERY_FORMATS)
query = {'format': format,
'query': query}
response = utils.urlread(_QUERY_URL, query)
return formats.parse(response, format, parse)
