Example #1
0
 def fetch_resource(self, r, key, is_binary, is_attrib_url):
     if is_attrib_url:
         resource_path = get_attribs_key_url(r.attrib, key)
     else:
         resource_path = r.attrib[key]
     if not (resource_path.startswith('http://') or resource_path.startswith('https://')):
         parsed = urlparse(self.page_url)
         resource_path = parsed.scheme + '://' + adjoin_paths(parsed.hostname, resource_path)
     for res in re.findall('/<!--.*?-->', resource_path): resource_path = resource_path.replace(res, '')
     resource_path = resource_path.replace('http:///', 'http://')
     resource_path = resource_path.replace('https:///', 'https://')
     print 'fetching :', resource_path
     file_content = get_markup(resource_path, source_url=self.page_url)
     print 'done     : %s' % resource_path
     parsed = urlparse(resource_path)
     internal_path = adjoin_paths(self.site_path, parsed.hostname, parsed.path)
     ensure_dir_exists(os.path.dirname(internal_path))
     if file_content: write_file(internal_path, file_content, is_binary)
     markup_path = self.markup_path(internal_path)
     if is_attrib_url:
         set_attribs_key_url(r.attrib, key, markup_path)
     else:
         r.attrib[key] = markup_path
     if internal_path.endswith('.css'):
         self.process_css_urls(internal_path, resource_path)
Example #2
0
 def _cl(r):
     if r.startswith('http://'): raise Exception('Unhandled css parse case')
     if r.startswith('https://'): raise Exception('Unhandled css parse case')
     online_path_file = adjoin_paths(os.path.dirname(online_css_path_file.split('?')[0]), r)
     online_file = os.path.basename(online_path_file).split('?')[0]
     local_path_file = adjoin_paths(os.path.dirname(local_css_path_file), r)
     ensure_dir_exists(os.path.dirname(local_path_file))
     print 'fetching :', online_file
     for res in re.findall('/<!--.*?-->', online_file): online_file = online_file.replace(res, '')
     file_content = get_markup(online_path_file, source_url=self.page_url)
     print 'done     : %s' % online_file
     if file_content: write_file(local_path_file, file_content, True)
Example #3
0
 def process_internal_asset(self, selector, asset_path, asset_file, t_tag, new_cl): 
     res = self.tree_root.xpath(selector)
     file_content = u'\n\n'.join(u'\n'.join([l for l in (r.text if r.text else u'').split('\n')]).strip() for r in res)
     for r in res: 
         remove_from_parent(r)
         r.text = None
         try: 
             assert r.attrib.has_key('src')
             append_to_tag(self.tree_root, 'head', r)
         except:
             remove_from_parent(r)
     unique_file_name = uniquify_file_name(asset_path, asset_file)
     internal_path = adjoin_paths(asset_path, unique_file_name)
     ensure_dir_exists(asset_path)
     write_file(internal_path, file_content)
     append_to_tag(self.tree_root, t_tag, new_cl(self.markup_path(internal_path)))
     if asset_file.endswith('.css'):
         self.process_css_urls(internal_path, self.page_url)
Example #4
0
    def main_logic(self, open_when_done=False):
        print 'getting resources for page  : %s' % self.page_url
        # process links, script sources and images, flash and other media
        if self.fetch_resouces:
            self.fetch_external_resources()

        # wait for all blocking IO threads to finish fetching external resources
        self.pool.waitall()

        # lorem ipsify text
        if self.randomize_text:
            self.process_text()

        # process imbedded css
        if self.process_embedded_css:
            self.process_internal_asset('//style', adjoin_paths(os.path.dirname(self.index_path), 
                                                            'imbedded_css'), 'imbedded.css', 'head', create_css_link)     
        
        # process inline js
        if self.process_inline_js:
            self.process_internal_asset('//body//script', adjoin_paths(os.path.dirname(self.index_path), 
                                                            'inline_js'), 'inline.js', 'body', create_js_tag)
        
        # remove comments
        if self.remove_comments:
            self.filter_comments(True)
        
        # process noscript tags
        if self.remove_ns_tags:
            self.process_noscript_tags()
        
        # render html
        html_output = render_html_element(self.tree_root, format_html5=self.use_html5)
        #print html_output
        
        ensure_dir_exists(os.path.dirname(self.index_path))
        write_file(self.index_path, html_output)

        print 'done getting resources      : %s' % self.page_url
        if open_when_done: open_in_browser(self.index_path)