def main(): parser = argparse.ArgumentParser(description="Exchange DB converter") parser.add_argument("--dbfile", type=str, default='vkexch.sqlite', help="Path to sqlite DB file") parser.add_argument("--csvfile", type=str, default='vkexch.csv', help="Path to csv file") args = parser.parse_args() exchangeDB.bind('sqlite', args.dbfile, create_db=True) exchangeDB.generate_mapping(create_tables=True) def str2(s): return s if s else 'None' with db_session, codecs.open(args.csvfile, 'w', encoding='utf-8', errors='replace') as csv_file: for public in select(public for public in Public): csv_file.write('\t'.join([str(public.club_id), str(public.public_id), \ str(public.size), str(public.price), str(public.coverage), \ str(public.coverage_day), str2(public.name), str2(public.category)])) csv_file.write('\n')
def main(): logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') logging.getLogger("selenium").setLevel(logging.WARNING) parser = argparse.ArgumentParser(description="Sociate exchange collector") parser.add_argument("--dbfile", type=str, default='scexch.sqlite', help="Path to sqlite DB file") args = parser.parse_args() exchangeDB.bind('sqlite', args.dbfile, create_db=True) exchangeDB.generate_mapping(create_tables=True) #driver = selenium.webdriver.Firefox() #driver = selenium.webdriver.PhantomJS() dcap = dict(DesiredCapabilities.PHANTOMJS) dcap["phantomjs.page.settings.userAgent"] = USER_AGENT driver = selenium.webdriver.PhantomJS(desired_capabilities=dcap) logging.info('Collecting Sociate exchange info...') collect_exchange(driver) logging.info('Exchange collected') driver.close()
def main(): logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') logging.getLogger("selenium").setLevel(logging.WARNING) parser = argparse.ArgumentParser(description="VK exchange collector") parser.add_argument("--dbfile", type=str, default='vkexch.sqlite', help="Path to sqlite DB file") parser.add_argument("--cksfile", type=str, default='vkcookies.json', help="Path to cookies file") parser.add_argument("--authfile", type=str, default='.vkauth', help="Path to cookies file") args = parser.parse_args() exchangeDB.bind('sqlite', args.dbfile, create_db=True) exchangeDB.generate_mapping(create_tables=True) #driver = selenium.webdriver.Firefox() #driver = selenium.webdriver.PhantomJS() dcap = dict(DesiredCapabilities.PHANTOMJS) dcap["phantomjs.page.settings.userAgent"] = USER_AGENT driver = selenium.webdriver.PhantomJS(desired_capabilities=dcap) # We have to navigate to VK page before setting cookies # (PhantomJS' feature, see https://github.com/detro/ghostdriver/issues/178) driver.get("https://vk.com") try: cookies = json.load(open(args.cksfile, "r")) for cookie in cookies: driver.add_cookie(cookie) except: logging.info('Autorization...') vk_auth_info = json.load(open(args.authfile)) vk_auth(driver, vk_auth_info['username'], vk_auth_info['password']) logging.info('Autorization passed') json.dump(driver.get_cookies(), open(args.cksfile, 'w')) logging.info('Collecting VK exchange info...') collect_exchange(driver) logging.info('Exchange collected') driver.close()