def main():
parser = argparse.ArgumentParser(description="Exchange DB converter")
parser.add_argument("--dbfile", type=str, default='vkexch.sqlite', help="Path to sqlite DB file")
parser.add_argument("--csvfile", type=str, default='vkexch.csv', help="Path to csv file")
args = parser.parse_args()
exchangeDB.bind('sqlite', args.dbfile, create_db=True)
exchangeDB.generate_mapping(create_tables=True)
def str2(s):
return s if s else 'None'
with db_session, codecs.open(args.csvfile, 'w', encoding='utf-8', errors='replace') as csv_file:
for public in select(public for public in Public):
csv_file.write('\t'.join([str(public.club_id), str(public.public_id), \
str(public.size), str(public.price), str(public.coverage), \
str(public.coverage_day), str2(public.name), str2(public.category)]))
csv_file.write('\n')
def main():
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
logging.getLogger("selenium").setLevel(logging.WARNING)
parser = argparse.ArgumentParser(description="Sociate exchange collector")
parser.add_argument("--dbfile", type=str, default='scexch.sqlite', help="Path to sqlite DB file")
args = parser.parse_args()
exchangeDB.bind('sqlite', args.dbfile, create_db=True)
exchangeDB.generate_mapping(create_tables=True)
#driver = selenium.webdriver.Firefox()
#driver = selenium.webdriver.PhantomJS()
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = USER_AGENT
driver = selenium.webdriver.PhantomJS(desired_capabilities=dcap)
logging.info('Collecting Sociate exchange info...')
collect_exchange(driver)
logging.info('Exchange collected')
driver.close()
def main():
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
logging.getLogger("selenium").setLevel(logging.WARNING)
parser = argparse.ArgumentParser(description="VK exchange collector")
parser.add_argument("--dbfile", type=str, default='vkexch.sqlite', help="Path to sqlite DB file")
parser.add_argument("--cksfile", type=str, default='vkcookies.json', help="Path to cookies file")
parser.add_argument("--authfile", type=str, default='.vkauth', help="Path to cookies file")
args = parser.parse_args()
exchangeDB.bind('sqlite', args.dbfile, create_db=True)
exchangeDB.generate_mapping(create_tables=True)
#driver = selenium.webdriver.Firefox()
#driver = selenium.webdriver.PhantomJS()
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = USER_AGENT
driver = selenium.webdriver.PhantomJS(desired_capabilities=dcap)
# We have to navigate to VK page before setting cookies
# (PhantomJS' feature, see https://github.com/detro/ghostdriver/issues/178)
driver.get("https://vk.com")
try:
cookies = json.load(open(args.cksfile, "r"))
for cookie in cookies:
driver.add_cookie(cookie)
except:
logging.info('Autorization...')
vk_auth_info = json.load(open(args.authfile))
vk_auth(driver, vk_auth_info['username'], vk_auth_info['password'])
logging.info('Autorization passed')
json.dump(driver.get_cookies(), open(args.cksfile, 'w'))
logging.info('Collecting VK exchange info...')
collect_exchange(driver)
logging.info('Exchange collected')
driver.close()
