from time import sleep import re import pandas as pd import sys import os import csv import pickle from random import shuffle sys.path.insert(0, './Functions_scripts') # Allows to set a different path for the scripts being called below (but only if it branches off of the root dir) from downloaddate_function import longtime, downloaddate from loggenerator import gen_log pd.set_option('display.max_columns', 500) #######Initialization####### log_starttime = longtime() # Get the start time for the log file projpath = './' datapath = projpath + 'data_raw/' input_path = datapath + 'Register_scrape/' output_path = datapath + 'Scrape/' if not os.path.exists(output_path): # If the path doesn't exist, make it os.makedirs(output_path) #######Main program###### #Input file - read in char links input_file = input_path + 'ca_char_list.csv' with open(input_file, 'r', encoding='1252', errors='replace', newline='') as inputfile: # At the end, this reads the output file back in and checks it contains the same number of records as the target df = pd.read_csv(inputfile)
import pandas as pd import sys import os import csv sys.path.insert( 0, './Functions_scripts' ) # Allows to set a different path for the scripts being called below (but only if it branches off of the root dir) from downloaddate_function import longtime, downloaddate from loggenerator import gen_log #######Toggles####### start_fresh = False # This, if set False, allows the scraper to pick up where it left off if it failed during a run. If set true it will overwrite any previous results and start from page 1. sleeptime = 5 # This is how long to sleep between each page grab - 5 seconds makes the scraper take about 5.5 hours in total. #######Initialization####### log_starttime = longtime() # Get the start time for the log file projpath = './' datapath = projpath + 'data_raw/Register_scrape/' if not os.path.exists(datapath): # If the path doesn't exist, make it os.makedirs(datapath) baseurl = 'https://apps.cra-arc.gc.ca/ebci/hacc/srch/pub/advncdSrch?dsrdPg=' # This is the URL for the search of all charities - it just need a page number between the base and suffix suffixurl = '&q.stts=0007&q.ordrClmn=NAME&q.ordrRnk=ASC' #######Main program###### #Output file output = datapath + 'ca_char_list.csv' if not os.path.exists(
print('New proxies grabbed.\n') return proxies #######Main program####### """ # Fetch Dropbox authentication dbtokenpath = 'C:/Users/mcdonndz-local/Desktop/admin/db_token.txt' #dbtokenpath_pi = '/home/pi/admin/dp_token.txt' dbtokenfile = open(dbtokenpath, "r") dbapitoken = dbtokenfile.read() print(dbapitoken) dbx = dropbox.Dropbox(dbapitoken) # Create an object for accessing Dropbox API """ log_starttime = longtime() # Get the current date # Define paths projpath = './' # Location of syntax localdatapath = projpath + 'temp/' # The location of the input file changes based on the month it was downloaded - move it manually into this temp folder if not os.path.exists(localdatapath): # If the paths don't exist, make them os.makedirs(localdatapath) downloadpath = 'ew_download/Trustee_scrape' if not os.path.exists(downloadpath): os.makedirs(downloadpath) #Set the filenames inputfile = localdatapath + 'extract_main_charity.csv' # This file is downloaded by ew_download.py outputfilename = '/' + 'ew_trustee_data' + '.csv' # This is the main output data outputfile = projpath + downloadpath + outputfilename status = downloadpath + '/Status.txt' # Status is a text file which records the last charity which the scraper atempted to parse, can be used for debugging failue. This is automatically removed when the script completes.