def download_data(self): BreweryDb.configure('c7aeca2c6fa21f8a51e023c80332a870') response = self.method_to_invoke() totalPages = response.get('numberOfPages', None) if totalPages: print 'Total Page to read %d : ' % totalPages response_data = response['data'] model_dict_list = self.parse_data(response_data, self.get_model_props()) variables = self.get_model_props() hops_count = 0 with open(self.file_name(), 'w') as csvFile: writer = DictUnicodeWriter(csvFile, variables) print "Wrote header" writer.writeheader() writer.writerows(model_dict_list) hops_count += len(model_dict_list) print "Wrote %d records to file " % hops_count if totalPages: for page in range(2, totalPages + 1, 1): print "Retrieving data for page %d of total %d" % (page, totalPages) response = self.method_to_invoke_with_params({'p': page}) model_dict_list = self.parse_data(response['data'], variables) self.write_to_csv(self.file_name(), variables, model_dict_list) hops_count += len(model_dict_list) print "Wrote %d to file " % hops_count
def download_beers(): # response = BreweryDb.beers({'withBreweries':'Y','withIngredients':'Y'}) response = BreweryDb.beers({'withBreweries': 'Y'}) totalPages = response['numberOfPages'] beers_data = response['data'] beer_dict_list = parse_beer_data(beers_data) variables = [x for x in dir(Beer) if not (x.startswith('_'))] beer_count = 0 print "Starting to process page 1 of %d " % totalPages with open('beers-brewerydb.csv', 'w') as csvFile: writer = DictUnicodeWriter(csvFile, variables) print "Wrote header" writer.writeheader() writer.writerows(beer_dict_list) beer_count += len(beer_dict_list) print "Wrote %d beers to file " % beer_count for page in range(2, totalPages + 1, 1): print "Retrieving data for page %d of total %d" % (page, totalPages) response = BreweryDb.beers({'withBreweries': 'Y', 'p': page}) beer_dict_list = parse_beer_data(response['data']) write_beers_to_csv('beers-brewerydb.csv', variables, beer_dict_list) beer_count += len(beer_dict_list) print "Wrote %d beers to file " % beer_count
def download_yeast(): BreweryDb.configure('c7aeca2c6fa21f8a51e023c80332a870') # response = BreweryDb.beers({'withBreweries':'Y','withIngredients':'Y'}) response = BreweryDb.yeasts() totalPages = response['numberOfPages'] hops_data = response['data'] hops_dict_list = parse_hops_data(hops_data) variables = [x for x in dir(Yeast) if not (x.startswith('_'))] hops_count = 0 print "Starting to process page 1 of %d " % totalPages with open('yeast-brewerydb.csv', 'w') as csvFile: writer = DictUnicodeWriter(csvFile, variables) print "Wrote header" writer.writeheader() writer.writerows(hops_dict_list) hops_count += len(hops_dict_list) print "Wrote %d beers to file " % hops_count for page in range(2, totalPages + 1, 1): print "Retrieving data for page %d of total %d" % (page, totalPages) response = BreweryDb.yeasts({'p': page}) beer_dict_list = parse_hops_data(response['data']) write_hops_to_csv('yeast-brewerydb.csv', variables, beer_dict_list) hops_count += len(beer_dict_list) print "Wrote %d yeasts to file " % hops_count
def download_categories(): BreweryDb.configure('c7aeca2c6fa21f8a51e023c80332a870') # response = BreweryDb.beers({'withBreweries':'Y','withIngredients':'Y'}) response = BreweryDb.categories() cat_data = response['data'] variables = [x for x in dir(Category) if not (x.startswith('_'))] cat_dicts = parse_cat_data(cat_data) with open('categories-brewerydb.csv', 'w') as csvFile: writer = DictUnicodeWriter(csvFile, variables) print "Wrote header" writer.writeheader() writer.writerows(cat_dicts) print "Wrote %d categories to file " % len(cat_dicts)
def download_hops(): BreweryDb.configure('c7aeca2c6fa21f8a51e023c80332a870') # response = BreweryDb.beers({'withBreweries':'Y','withIngredients':'Y'}) response = BreweryDb.hops() totalPages = response['numberOfPages'] hops_data = response['data'] hops_dict_list = parse_hops_data(hops_data) variables = [x for x in dir(Hops) if not (x.startswith('_'))] hops_count = 0 print "Starting to process page 1 of %d " % totalPages with open('hops-brewerydb.csv', 'w') as csvFile: writer = DictUnicodeWriter(csvFile, variables) print "Wrote header" writer.writeheader() writer.writerows(hops_dict_list) hops_count += len(hops_dict_list) print "Wrote %d beers to file " % hops_count for page in range(2, totalPages + 1, 1): print "Retrieving data for page %d of total %d" % (page, totalPages) response = BreweryDb.beers({'p': page}) beer_dict_list = parse_hops_data(response['data']) write_hops_to_csv('beers-brewerydb.csv', variables, beer_dict_list) hops_count += len(beer_dict_list) print "Wrote %d beers to file " % hops_count
def write_beers_to_csv(file_name, variables, beer_dict_list): with open(file_name, 'a') as csvFile: writer = DictUnicodeWriter(csvFile, variables) writer.writerows(beer_dict_list)
def write_hops_to_csv(file_name, variables, beer_dict_list): with open(file_name, 'a') as csvFile: writer = DictUnicodeWriter(csvFile, variables) writer.writerows(beer_dict_list)
def write_to_csv(self, file_name, variables, model_dict_list): with open(file_name, 'a') as csvFile: writer = DictUnicodeWriter(csvFile, variables) writer.writerows(model_dict_list)