def main(): try: delete_old_results() except Exception as e: print("error:", e) try: launch = visit_url() if navigator(launch): print("Completed !") except Exception as e: print("error:", e) print('started generate report...') sleep(60) try: first_modify(output_config.get('json_file')) except Exception as e: print("error:", e) try: rewrite(output_config.get('json_file')) except Exception as e: print("error:", e) try: jsontocsv(output_config.get('jsonreport')) except Exception as e: print("error:", e)
item.update({"product review": "not present"}) try: item.update({ "recommends": soup.find('span', attrs={ 'class': 'messaging' }).text.strip() }) except Exception as e: item.update({"recommends": "not present"}) try: item.update({ 'shipping': ship.split('\n') for ship in soup.find('li', attrs={ 'class': 'region' }).text.split('\n\n') if 'Shipping' in ship }) except Exception as e: item.update({'shipping': "not present"}) finally: return item def stop(): session.close() if __name__ == "__main__": product_page_link = output_config.get('product_link') print(start(product_page_link))
def delete_old_results(): results_folder = output_config.get('results') for file_old in os.listdir(results_folder): files = os.path.join(results_folder, file_old) print("Deleted:{}".format(files)) os.remove(files)
import json import io from resources.results import output_config try: to_unicode = unicode except NameError: to_unicode = str json_filename = output_config.get('json_file') class JsonWriter: def __init__(self): self.file_json = json_filename # self.first_line() def first_line(self): with open(self.file_json, 'a+') as file_writer: file_writer.write('[') def json_writer(self, dict_value): with io.open(self.file_json, 'a+', encoding='utf8') as outfile: str_ = json.dumps(dict_value, indent=4, sort_keys=True, separators=(',', ': '), ensure_ascii=False) outfile.write(to_unicode(str_)) outfile.write(",\n") return True
def jsontocsv(json_report): try: df = pd.read_json(json_report, orient='columns') df.to_csv(output_config.get('csv_file')) except ValueError as ve: print("Error: Json file not proper.\nplease check in reports folder")
import pandas as pd from resources.results import output_config import os json_report = output_config.get('jsonreport') def first_modify(file_json): with open(file_json, 'a+') as file_writer: file_writer.seek(0, os.SEEK_END) # seek to end of file; f.seek(0, 2) is legal file_writer.seek(file_writer.tell() - 3, os.SEEK_SET) # go backwards 3 bytes file_writer.truncate() file_writer.write("]") def rewrite(json_file): with open(json_file,errors='ignore',encoding="utf8") as read_json: with open(json_report,'a+',errors='ignore',encoding="utf8") as json_writer: json_writer.write("[\n") for line in read_json.readlines(): json_writer.write(line) def jsontocsv(json_report): try: df = pd.read_json(json_report, orient='columns') df.to_csv(output_config.get('csv_file')) except ValueError as ve: print("Error: Json file not proper.\nplease check in reports folder")
class NextVariables: next_filename = output_config.get('pages') product_filename = output_config.get('product')