def hello_world(): league_id = request.args.get('league_id') if league_id: return render_template('index.html', data=scrape(league_id)) # return render_template('index.html', data=['test', 'test2', 'test3']) else: return render_template('formTemplate.html')
def hello_world(): league_id = request.args.get('league_id') if league_id: return render_template('index.html', data=scrape(league_id)) # return render_template('index.html', data=['test', 'test2', 'test3']) else: return render_template('formTemplate.html')
def similarity_score(self, query, url): page = scrape(Site(url, True)) keywords = get_keywords(page.text) score = 0 for word in query: if word in keywords: score += 1 return (score)
def main(): # Program execution starts here. url = "http://sites.cs.queensu.ca/courses/cisc121/a2/logons.txt" print(f"Retrieving data from {url}.") data = web_scraper.scrape(url) n = len(data) print(f"{n} records read.") # This loop converts the data from a list of strings to a list of lists. for i in range(n): data[i] = line_to_list(data[i]) # CSV file creation # This code first opens a file with the requested name, # then joins all values in each interior array with commas and # makes a newline between each array, writing the result into the csv. csv_filename = "logons.csv" csv_file = open(f"{csv_filename}", "w") for i in data: csv_file.write(",".join([str(x) for x in i]) + "\n") csv_file.close() # JSON file creation # Note: Once your dict_make.make_dict() function is working, # this code will create a JSON file for you. You don't need # to do anything in this section. json_filename = "logons.json" dict_keys = ["user_id", "os", "pc_id", "datetime"] dict_list = [] for i in range(n): dict_list.append(dict_maker.make_dict(dict_keys, data[i])) if json_writer.write_json_file(json_filename, dict_list): print(f"Write to {json_filename} complete.") else: print(f"Write to {json_filename} failed.") while True: # This code checks all entries in data for the input PC id, # if it finds a match, it checks if the operating system is Linux. # In cases where both conditions are met, adds 1 to the Linux login count, # and prints the count after the whole list has been checked. print("Checking the number of Linux logons for a specific PC.") pc_id = input("Enter the PC's ID number or just press Enter to quit: ") logon_count = 0 if pc_id == "": break for i in range(n): if data[i][2] == pc_id: if data[i][1] == "Linux": logon_count = logon_count + 1 print(f"PC {pc_id} was logged onto in Linux {logon_count} times.") print("Done.")
def populate_web(self): """ Populates the web and the inverted index keyword dictionary with the urls provided """ occurdic = {} for url in self.urls: page = scrape(url) keywords = get_keywords(page.text) index = len(self.web) self.web.append(Index(index, page.title, page.url, page.links_to)) for word in keywords: value = OccurrenceList() value.append(index) occurdic[word.lower()] = value self.keywords.add(word.lower(), value) self.rank_page(occurdic, len(self.web))
update, frames=num_of_iterations, repeat=False) canvas.draw() # Only run if script is called directly. if __name__ == '__main__': ## Read parameters from command line ## (if none set, will set defaults as defined in read_cmd.py). parameters = read_cmd.parameters(argv) num_of_rabbits, lifespan, neighbourhood, num_of_iterations, animate = parameters ## Scrape initial x- and y-values from webpage. url = 'http://www.geog.leeds.ac.uk/courses/computing/practicals/python/agent-framework/part9/data.html' scraped_coordinates = web_scraper.scrape(url) ## Create environment from CSV file. environment = Environment("in.txt").env ## Create rabbits. rabbits = create_rabbits(environment, num_of_rabbits, scraped_coordinates, lifespan) # Configure Tkinter. root = tkinter.Tk() root.wm_title("Model") fig = matplotlib.pyplot.figure(figsize=(7, 7)) canvas = matplotlib.backends.backend_tkagg.FigureCanvasTkAgg(fig, master=root) canvas._tkcanvas.pack(side=tkinter.TOP, fill=tkinter.BOTH, expand=1)
if __name__ == '__main__': ## Read parameters from command line ## (if none set, will set defaults as defined in read_cmd.py). parameters = read_cmd.parameters(argv) num_of_rabbits, lifespan, neighbourhood, num_of_iterations, animate = parameters report = f"""##### Initial Parameters ##### Number of rabbits: {num_of_rabbits} Lifespan: {lifespan} Number of Iterations: {num_of_iterations} """ print(report) ## Scrape initial x- and y-values from webpage. URL = 'http://www.geog.leeds.ac.uk/courses/computing/practicals/python/agent-framework/part9/data.html' scraped_coordinates = web_scraper.scrape(URL) print(f"You have successfuly scraped coordinates from {URL}.") ## Create environment from CSV file. file = "in.txt" environment = agentframework.Environment(file).env print(f"Environment successfully created from {file}.") ## Create rabbits. rabbits = create_rabbits(environment, num_of_rabbits, scraped_coordinates, lifespan) print(f"You have successfuly created {num_of_rabbits} rabbits.") ## Run model and time it. print("We are now going to run the model.") time_result = run_model()
def main(): """Program execution starts here.""" url = "http://sites.cs.queensu.ca/courses/cisc121/a2/logons.txt" data = web_scraper.scrape(url) n = len(data) data = parse_logons.logons_to_list_of_dicts(data) # The data list (and the file it came from) are in chronological # order. That means data is currently sorted by timestamp. sort_on_key = "timestamp" main_menu_title = "Select an operation:" main_menu_choices = [ "Sort logons data", "Show a selection of the data", f"Search for a particular {sort_on_key}" ] sort_on_menu_title = "Select a key to sort on:" sort_on_menu_choices = list(data[0].keys()) sort_menu_title = "Select a sort algorithm:" sort_menu_choices = [ "Insertion sort", "Bubble sort", "Bubble sort (optimized)", "Selection sort" ] while True: choice = menu.do_menu(main_menu_title, main_menu_choices) if choice is None: return # Exit main() (and program). if choice == 1: # Sort logons data. while True: sort_on_choice = menu.do_menu(sort_on_menu_title, sort_on_menu_choices) if sort_on_choice is None: break # Return to main menu. sort_on_key = sort_on_menu_choices[sort_on_choice - 1] # Change last choice in main menu to reflect the new # sort_on_choice. main_menu_choices[ -1] = f"Search for a particular {sort_on_key}" sort_choice = menu.do_menu(sort_menu_title, sort_menu_choices) if sort_choice is None: break # Return to main menu. # If we're here, we can proceed with a sort. print() if sort_choice == 1: # Insertion sort sort_function = dict_quad_sorts.insertion_sort elif sort_choice == 2: # Bubble sort sort_function = dict_quad_sorts.bubble_sort elif sort_choice == 3: # Bubble sort (opt) sort_function = dict_quad_sorts.bubble_sort_opt else: # Selection sort sort_function = dict_quad_sorts.selection_sort # Do the sort. print(f"Sorting on key '{sort_on_key}'...") sort_function(data, sort_on_key) print("Done.") # Show it worked. long_list_printer.print_list(data, 5) elif choice == 2: # Show a selection of the data. long_list_printer.print_list(data, 10) elif choice == 3: # Search for a specific value. search_val = input(f"\nSearch for what {sort_on_key}? ") found_at = dict_bin_search.search(data, sort_on_key, search_val) if found_at is None: print(f"{sort_on_key} {search_val} not found.") else: print(f"{sort_on_key} {search_val} found at position "\ f"{found_at}.")
from web_scraper import scrape from xlsx_writer import save_file try: scrape() finally: save_file()
def main(): if args.mode == "scan": if "books.csv" not in os.listdir(root) or "books" not in os.listdir( root): sys.exit( "\nThere is no data to scan. Run the script with scrape mode first." ) else: try: start = time.time() mapper = WordMapper( ) # Creating the WordMapper object loads every word in the disk to ram. end = time.time() logging.info( "{} words are loaded into memory in {} seconds.".format( mapper.word_count(), (end - start))) mapper.map_words() # This will create mappings for every word. prefixes, suffixes = WordMapper.stats() logging.info("{} words are mapped to {} suffixes.".format( prefixes, suffixes)) os.chdir("..") with open(args.output, "wb") as output: pickle.dump( mapper.word_mappings, output, pickle.HIGHEST_PROTOCOL) # Saving the mappings. logging.info("Mappings written to {} in {}.".format( args.output, os.getcwd())) except KeyboardInterrupt: sys.exit(logging.error("Exiting current process.")) elif args.mode == "scrape": if "books.csv" not in os.listdir( root): # If books.csv doesn't exist, creating it first. logging.info("Writing books.csv ") top_book_urls = web_scraper.scrape() # Getting the list of urls. df = web_scraper.to_df(top_book_urls) # Adding them to df. web_scraper.save(df) # Saving df to csv. else: # If books.csv exists, but books directory doesn't exist. df_books = pd.read_csv("books.csv") if not os.path.exists( web_scraper.BOOKS_DIR ): # Creating the directory, if it doesn't exist. os.makedirs(web_scraper.BOOKS_DIR) count = 0 for i in df_books.index: # Iterating the .csv file and writing books to disk. if df_books.iloc[i, 0] + ".txt" in os.listdir( web_scraper.BOOKS_DIR): logging.error("{} is already written to the disk.".format( df_books.iloc[i, 0])) else: try: try: web_scraper.write_book(df_books.iloc[i, 1], df_books.iloc[i, 0]) count += 1 except TypeError: logging.error( "Unable to write {} to the disk.\n".format( df_books.iloc[i, 0])) except KeyboardInterrupt: logging.info( "{} books written to the disk in the current process.\n" .format(count)) sys.exit( logging.error( "There are {} books in the disk.\n".format( sum(1 for _ in os.listdir( web_scraper.BOOKS_DIR))))) elif args.mode == "chain": try: with open(args.input, "rb") as input_file: mapping = pickle.load(input_file) chain = word_chainer.chain_words(mapping, args.word) print(chain) except FileNotFoundError: sys.exit( logging.error("File {} is not found in {}.".format( args.input, os.getcwd())))