def test_csv_parser_with_blank_file(): global test_file # value missing with open(test_file,"w")as f: f.write("") y = CsvParser(test_file) y.load_file() y.parse_file() y.process_file() assert(y.get_value() == 0)
def _get_vehicle_and_passes_from_csv( path_to_csv: str) -> typing.Tuple[Vehicle, list]: """Get data to be analyzed, will be parsed into a Vehicle object as well as a list of passes Args: path_to_csv (str) Returns: typing.Tuple[Vehicle, list] """ cp = CsvParser() licence_plate, passes = cp.parse_csv(path_to_csv) return _get_vehicle_from_licence_plate(licence_plate), passes
def test_csv_parser_with_no_child_nodes(): global test_file # value missing with open(test_file,"w")as f: f.write("""name,active,value""") y = CsvParser(test_file) y.load_file() y.parse_file() y.process_file() assert(y.get_value() == 0) # delete the test file as not required os.remove(test_file)
def test_csv_parser_with_non_csv_values(): global test_file # value missing with open(test_file,"w")as f: f.write("""name,active,value John true Mark true Paul false 100 Ben true 150 """) y = CsvParser(test_file) y.load_file() y.parse_file() y.process_file() assert(y.get_value() == 0)
def test_csv_parser_with_missing_values(): global test_file # value missing with open(test_file,"w")as f: f.write("""name,active,value John,true, Mark,true, Paul,false,100 Ben,true,150 """) y = CsvParser(test_file) y.load_file() y.parse_file() y.process_file() assert(y.get_value() == 150)
def run(website_urls, outputfolder='', export_tabs=False, export_reports=False, export_bulk_exports=False): # Automatically create the correct --headless Screaming Frog commands; sf = ScreamingFrogAnalyser(website_urls=website_urls, outputfolder=outputfolder, export_tabs=export_tabs, export_reports=export_reports, export_bulk_exports=export_bulk_exports) # Start running the web crawls sf.run_crawls() parser = CsvParser(outputfolder=outputfolder, file_paths=sf._sf_folders, website_urls=sf._website_urls) # Return the objects for running tests; return sf
from csv_parser import CsvParser __author__ = 'hellfish90' if __name__ == '__main__': filename = "test_files/test_location.csv" parser = CsvParser(filename) header = parser.get_data_types() data_set = parser.get_set_by_data_and_location(1, 1) for item in header: print item, print "" for data in data_set[0]: print data['coordinates'], data['data'] print "Missed Rows: ", len(data_set[1]) parser.close_file()
def __init__(self): self._parser = {'.csv': CsvParser()} self.processor_factory = ProcessorFactory()
def sf_run(website_urls, outputfolder='', export_tabs=False, export_reports=False, export_bulk_exports=False, push_data_to_biquery=False, create_bigquery_table=False, bigquery_table_mapping=BIGQUERY_TABLE_ID_MAPPINGS): if OUTPUTFOLDER == '': raise ValidationError( 'Your OUTPUTFOLDER cannot be empty', 'Please update your outputfolder to a valid value.') # Automatically create the correct --headless Screaming Frog commands; sf = ScreamingFrogAnalyser(website_urls=website_urls, outputfolder=outputfolder, export_tabs=export_tabs, export_reports=export_reports, export_bulk_exports=export_bulk_exports) # 1. Start running + saving the web crawls sf.run_crawls() parser = CsvParser(outputfolder=outputfolder, file_paths=sf._sf_folders, website_urls=sf._website_urls) # 2.1 Data checking: Making sure that there is data & at least one of the dataframes contains rows: if not any(dataframe_checker(parser)): print( '''Finished crawling and saved the output to your desired folder/folders. It's impossible to save to BigQuery because you have no .csv data. Re-run the script with export_tabs, export_reports, or export_bulk_exports if you would like to upload to BigQuery! Existing the program. ''') # exit() <-- Disabling this whilst running tests. return sf # 2.1 Data checking - For valid credentials (Google Cloud Project ID + Service Account Key): if push_data_to_biquery == True: config_setup_check( [GOOGLE_CLOUD_PROJECT_ID, SERVICE_ACCOUNT_KEY_LOCATION]) # Google Cloud Credentials + BQ Client Initialisation credentials = service_account.Credentials.from_service_account_file( SERVICE_ACCOUNT_KEY_LOCATION) client = bigquery.Client(credentials=credentials, project=GOOGLE_CLOUD_PROJECT_ID) # 2.1 Data checking - Compile a list of dataframes that have both rows and columns: available_data = dataframe_row_checker(parser) # 3.1 Storing The Queryable Data: if create_bigquery_table == True: # Automatically generate the BigQuery tables with timestamped names + push the relevant data: print( "Some function here that will automatically generate Xn BigQuery tables." ) pass else: # Automatically use the BigQuery Table Mapping print( "Some function here that will map the name of the BigQuery table_id against the csv_name." ) if config._bigquery_inputs_validated == False: raise ValidationError( "You need to use a custom dictionary to map your concatenated .csv data against BigQuery table ids.", ''' Please update the setup.yaml file with the relevant bigquery_tab_id mappings.''' ) # Match the dictionary mapping against the available_data dictionary and only contain the Bigquery table_id's where there is data. # Error checking that the length of the dictionary keys are the same length as the available_data dict keys. pass # Return the objects for running tests; return sf
def test_parser_multiple_files_csvs(): parser = CsvParser(outputfolder=outputfolder, file_paths=csv_multiple_file_paths, website_urls=website_urls) # Multiple tests here: check_data_frame(parser)
def test_parser_multiple_files_no_csvs(): parser = CsvParser(outputfolder=outputfolder, file_paths=seo_spider_multiple_file_paths, website_urls=website_urls) assert len(parser._csv_data_dict.keys()) == 0
def setUp(self): self.csv_parser = CsvParser('1000 Sales Records.csv')
def setUp(self): self._csv_parser = CsvParser('testdata/csv_parser_test.csv', 'col3')
# prefer method enumerate(x) over range(len(x)) for product_index, product_name in enumerate(parser.product_names): cumulative_sales_per_product[product_index] += sales_per_week_per_product[product_index] number_of_records += 1 sales_report.total_sales_per_product = dict(zip(parser.product_names, cumulative_sales_per_product)) return number_of_records if __name__ == '__main__': # print("Test") # sales_text contains entire csv file filename = './data/sales.csv' with open(filename) as f: sales_text = f.read() # print(sales_text) parser = CsvParser(sales_text) sales_report = generate_sales_report(parser) print(sales_report.total_sales_per_week_report()) print(sales_report.week_with_highest_sales_report()) print(sales_report.total_sales_per_product_report()) # print(sales_report.total_sales_per_product_report_narrow_format()) print(sales_report.average_weekly_sales_report())
def run(): # Parse Command Line Options usage = "usage: %prog [options]" Parser = OptionParser(usage=usage) Parser.add_option("-f", "--file", dest="csv_file", metavar="STRING", help="CSV File Location") Parser.add_option("-c", "--config", dest="config_file", default="/etc/cobbler-csv.conf", metavar="STRING", help="Config file (default: /etc/cobbler-csv.conf)") (options, args) = Parser.parse_args() if len(sys.argv) == 1: Parser.print_help() sys.exit(1) config = Config(configFile=options.config_file) csv = CsvParser(options.csv_file, config) for system in csv: hostname = config.get_mapping("hostname").format(**system) print "Creating new system %s...\n" % hostname cobbler_system = CobblerSystem(options.config_file, hostname=hostname) interface = { 'interface': 'eth0', 'macaddress': config.get_mapping("macaddress").format(**system), 'ipaddress': config.get_mapping("ipaddress").format(**system), 'subnet': config.get_mapping("subnet").format(**system), 'gateway': config.get_mapping("gateway").format(**system), 'static': config.get_mapping("static").format(**system) } cobbler_system.set_interface(**interface) ns = ", ".join( config.get_mapping("name_servers").format(**system).split(" ")) search = config.get_mapping("name_servers_search").format(**system) kernel_opts = { 'hostname': hostname, 'ipaddress': interface['ipaddress'], 'netmask': interface['subnet'], 'gateway': interface['gateway'], 'nameserver': ns, 'search': search } cobbler_system.set_kernel_opts(**kernel_opts) attributes = [k[4:] for k in dir(cobbler_system) if k[0:4] == "set_"] for attribute in attributes: try: value = config.get_mapping(attribute).format(**system) getattr(cobbler_system, "set_" + attribute)(value) print "Setting %s:\n%s\n" % (attribute, value) except: continue # no biggie, not a required param cobbler_system.set_ks_meta(**dict([(k.lower().replace(" ", "_"), v) for k, v in system.iteritems()])) cobbler_system.save() print "System saved!" if config.update_custom_info: try: post_trigger(options.config_file, hostname=hostname) print "System custom info in Satellite updated!" except: pass print "-----------------------------------------------------------" if config.cobbler_sync: """ Sync cobbler """ print "Syncing cobbler..." cobbler_system.sync() print "Done." print "-----------------------------------------------------------"