def _create_file(self): """ Create the file and write an empty json structure """ with open(self.LOG_FILE, 'w') as f: f.write( json.dumps({ 'time': datetime.now().strftime(FilterXMLConfig.iso_fmt()), 'log': {} }))
def from_json(cls, row: dict) -> SmileyReport: """ Constructs a single SmileyReport object. Expects a dict as defined by a row in the smiley_reports list in: https://github.com/sw814f21/filter_xml#final-output """ self = SmileyReport() self.report_id = row['report_id'] self.smiley = int(row['smiley']) self.date = datetime.strptime(row['date'], FilterXMLConfig.iso_fmt()) return self
def from_json(cls, row: dict) -> Restaurant: """ Constructs a single Restaurant object Expects a dict as as defined by: https://github.com/sw814f21/filter_xml#final-output """ self = Restaurant() self.cvrnr = row['cvrnr'] self.pnr = row['pnr'] self.region = row['region'] self.industry_code = row['industry_code'] self.industry_text = row['industry_text'] self.start_date = datetime.strptime(row['start_date'], FilterXMLConfig.iso_fmt()) \ if row['start_date'] else '' self.end_date = datetime.strptime(row['end_date'], FilterXMLConfig.iso_fmt()) \ if row['end_date'] else '' self.smiley_reports = [ SmileyReport.from_json(report) for report in row['smiley_reports'] ] self.city = row['city'] self.elite_smiley = row['elite_smiley'] self.geo_lat = float(row['geo_lat']) if row['geo_lat'] else None self.geo_lng = float(row['geo_lng']) if row['geo_lng'] else None self.niche_industry = row['niche_industry'] self.url = row['url'] self.address = row['address'] self.name = row['name'].strip() if row['name'] else None self.name_seq_nr = row['name_seq_nr'] self.zip_code = row['zip_code'] self.ad_protection = row['ad_protection'] self.company_type = row['company_type'] self.franchise_name = row['franchise_name'] return self
def process_smiley_json(self, data: RestaurantCatalog) -> None: """ Processes smiley .json file. Includes only production units Applies filters from DataHandler Collects additional, external data through CVRHandler Restaurants that have already been processed (i.e., external data has been collected) are stored in processed_companies.csv - handled by PrevProcessedFile. Restaurants that have been processed during the current session are stored in temp.csv - handled by TempFile. This is done to save progress in the case of a crash during the run. Once data has been processed, keys are renamed. Cf. the translation map in _rename_keys() """ temp_file = TempFile() res = temp_file.get_all() total_rows = data.catalog_size row_index = 0 if self._cvr_handler.PRE_PROCESSING_STEP: self._cvr_handler.pre_processing(data.catalog) for restaurant in data.catalog: # we use this to avoid using the same fallback in three separate if statements row_kept = False # if sample size CLI arg is supplied, stop when its reached if self._sample_size and res.catalog_size >= self._sample_size: break # first check if the restaurant is valid if restaurant.is_valid_production_unit(): # then ensure it hasn't already been processed prior to a crash if not temp_file.contains(restaurant.name_seq_nr): # only sleep if --no-scrape is not passed, and if our cvr provider requests it. if not self._skip_scrape and self._cvr_handler.SHOULD_SLEEP and row_index > 0: time.sleep(self._cvr_handler.CRAWL_DELAY) # only collect data if we haven't passed --no-scrape if not self._skip_scrape: restaurant = self._cvr_handler.collect_data(restaurant) # check filters to see if we should keep the row # otherwise add it to blacklist so we don't scrape it next time if self.post_filters.filter(restaurant): if not self._skip_scrape: restaurant = self._smiley_handler.collect_data( restaurant) res.add(restaurant) row_kept = True temp_file.add_data(restaurant) else: Blacklist.add(restaurant) # if any check resulted in a row skip, decrement the total row count # for terminal output purposes if not row_kept: total_rows -= 1 if self._sample_size: if row_kept: print( f'Collected {res.catalog_size} of {self._sample_size} samples' ) else: print(f'{total_rows - res.catalog_size} rows to go') row_index += 1 self.post_filters.log_filters() token = datetime.now().strftime(FilterXMLConfig.iso_fmt()) res.setup_diff(self._outputter.get()) self._outputter.insert(res.insert_set(), token) self._outputter.update(res.update_set(), token) self._outputter.delete(res.delete_set(), token) temp_file.close() Blacklist.close_file()
def date_string(self) -> str: """ ISO-8601 formatted date string property """ return self.date.strftime(FilterXMLConfig.iso_fmt())
def end_date_string(self) -> str: """ ISO-8601 formatted start date string property """ return self.end_date.strftime( FilterXMLConfig.iso_fmt()) if self.end_date else ''