コード例 #1
0
def do_scrape(download_dir='download', download_processes_num=4, except_log_file_name='request_exceptions_fundays.log'):
    global img_dir
    global processes_num
    processes_num = download_processes_num
    img_dir = site_subdir_creator(img_dir)(download_dir)
    except_log_file = open(except_log_file_name, 'w')
    start_time = time.time()
    venues =  process_site_layers(site_layers_description_list, except_log_file)
    except_log_file.close()
    print 'test.fundays.ie scrapping time: ', str(time.time() - start_time)
    save_data(venues, 'fundays.dat')
    fundays_data = get_saved_data('fundays.dat')
    to_csv(fundays_data)
def do_scrape(
    download_dir="download", download_processes_num=8, except_log_file_name="request_exceptions_discoverireland.log"
):
    global img_dir
    global processes_num
    processes_num = download_processes_num
    img_dir = site_subdir_creator(img_dir)(download_dir)
    except_log_file = open(except_log_file_name, "w")
    start_time = time.time()
    discoverireland_data = process_site_layers(site_layers_description_list, except_log_file)
    print "discoverireland.ie scrapping time: ", str(time.time() - start_time)
    save_data(discoverireland_data, "discoverireland.dat")
    except_log_file.close()
    discoverireland_data = get_saved_data("discoverireland.dat")
    to_csv(discoverireland_data)