def getLecuresFor(self, url): src = urlread(url) div_tags = BS(src, parseOnlyThese=SS('div', {'class': 'medialisting'})) #attempt to parse normal page if len(div_tags) > 0: items = self.parse_normal_course(div_tags) else: items = self.parse_rm_course(src) return items
def extractMediaFile(self, url, preferArchive=False): ocw_page_url = url src = urlread(ocw_page_url) if preferArchive: archive = self.extractInternetArchiveFiles(src=src) if archive: return archive youtube_url = self.extractYouTubeFile(src=src) return youtube_url
def _query(query, title=None, titles=None, project=_DEFAULT_PROJECT, format=formats.JSON, parse=True): formats.check(format, _SUPPORTED_FORMATS) full_query = { 'format': format, 'action': 'query', 'titles': _get_titles(title, titles) } full_query.update(query) response = utils.urlread(_get_url(project), full_query) return formats.parse(response, format, parse)
def getAllCourse(self, includeSelected=False): src = urlread(self.courses_url) div_tags = BS(src, parseOnlyThese=SS('tr', {'class': re.compile('row|alt-row')})) #filter out classes that don't have full video lectures available if includeSelected: lectureFilter = re.compile('Video lectures|Selected video lectures') else: lectureFilter = 'Video lectures' video_divs = filter(lambda d: d.find('a', {'title': lectureFilter}), div_tags) items = [{'name': div.u.string, 'url': self.urljoin(div.a['href']), 'mode': MODE_MIT_LECTURES} for div in video_divs] return items
def build_initrd(self,initrd,kickstart,data): """ Crack open an initrd and install the kickstart file. """ # save kickstart to file ksdata = utils.urlread(kickstart) fd = open("/var/spool/koan/ks.cfg","w+") if ksdata is not None: fd.write(ksdata) fd.close() # handle insertion of kickstart based on type of initrd fd = open("/var/spool/koan/insert.sh","w+") fd.write(self.get_insert_script(initrd)) fd.close() utils.subprocess_call([ "/bin/bash", "/var/spool/koan/insert.sh" ]) shutil.copyfile("/var/spool/koan/initrd_final", initrd)
def get_entity(id, version=None, format=formats.JSON, parse=True): """Retrieve entity data from WikiData. See https://www.wikidata.org/wiki/Wikidata:Data_access#Linked_Data_interface Args: id: The entity ID (str) version: The entity version (str or int, optional) format: The response format (str, optional) parse: Whether to parse the response (boolean, optional) Returns: The response (str or object) """ formats.check(format, _GET_ENTITY_FORMATS) url = _GET_ENTITY_BASE_URL + id + '.' + format query = {'version': str(version)} if version else None response = utils.urlread(url, query) return formats.parse(response, format, parse)
def get_install_tree_from_kickstart(self,profile_data): """ Scan the kickstart configuration for either a "url" or "nfs" command take the install_tree url from that """ try: raw = utils.urlread(profile_data["kickstart"]) lines = raw.splitlines() method_re = re.compile('(?P<urlcmd>\s*url\s.*)|(?P<nfscmd>\s*nfs\s.*)') url_parser = opt_parse.OptionParser() url_parser.add_option("--url", dest="url") nfs_parser = opt_parse.OptionParser() nfs_parser.add_option("--dir", dest="dir") nfs_parser.add_option("--server", dest="server") for line in lines: match = method_re.match(line) if match: cmd = match.group("urlcmd") if cmd: (options,args) = url_parser.parse_args(cmd.split()[1:]) profile_data["install_tree"] = options.url break cmd = match.group("nfscmd") if cmd: (options,args) = nfs_parser.parse_args(cmd.split()[1:]) profile_data["install_tree"] = "nfs://%s:%s" % (options.server,options.dir) break if self.safe_load(profile_data,"install_tree"): print "install_tree:", profile_data["install_tree"] else: print "warning: kickstart found but no install_tree found" except: # unstable to download the kickstart, however this might not # be an error. For instance, xen FV installations of non # kickstart OS's... pass
def _get(parameters, format=formats.JSON, parse=True): formats.check(format, _SUPPORTED_FORMATS) response = utils.urlread(_WIKIMEDIA_API_URL + '/'.join(parameters)) return formats.parse(response, format, parse)
def query(query, format=formats.JSON, parse=True): formats.check(format, _QUERY_FORMATS) query = {'format': format, 'query': query} response = utils.urlread(_QUERY_URL, query) return formats.parse(response, format, parse)