def download_menu_files(): menu_files = [] site_url = "http://hleb-sol.biz" wpp = WebPageProcessor() wpp.add_action( _type=Action.AT_FAST_DOWNLOAD, _name="webpage", _subject=site_url ) wpp.add_action( _type=Action.AT_PARSE_BY_SELECTOR, _name="link", _target="webpage", _subject=".menuItemBig td a[href]", _visible=True ) wpp.run() links = wpp.get_result() for lind, link in enumerate(links): rel_url = link.get('link') abs_url = site_url + rel_url upload_dir = '/tmp/force/' if not os.path.exists(upload_dir): os.makedirs(upload_dir) filename = u'{dt}-{di}.xls'.format(dt=unicode(timezone.now()), di=lind) urllib.urlretrieve(abs_url, upload_dir+filename) menu_files.append(upload_dir+filename) return menu_files
def tearDown(self): print('\n%s starts here' % self._testMethodName) start_time = time.time() w2p = WebPageProcessor() config = self.config.get('comments') for action in deepcopy(config): w2p.add_action(_type=action.get('type'), _data=action.get('data'), _name=action.get('name'), _target=action.get('target'), _settings=action.get('settings')) try: w2p.run() finally: result = w2p.get_result() errors = w2p.get_errors() warnings = w2p.get_warnings() info = w2p.get_info() w2p.free() print('%s executed in %05.2f seconds' % (self._testMethodName, (time.time() - start_time))) self.save_result(self._testMethodName, result) self.save_config(self._testMethodName, self.config) self.save_errors(self._testMethodName, errors) self.save_warnings(self._testMethodName, warnings) self.save_info(self._testMethodName, info) self.assertEqual(len(errors), 0)