def main():
try:
delete_old_results()
except Exception as e:
print("error:", e)
try:
launch = visit_url()
if navigator(launch):
print("Completed !")
except Exception as e:
print("error:", e)
print('started generate report...')
sleep(60)
try:
first_modify(output_config.get('json_file'))
except Exception as e:
print("error:", e)
try:
rewrite(output_config.get('json_file'))
except Exception as e:
print("error:", e)
try:
jsontocsv(output_config.get('jsonreport'))
except Exception as e:
print("error:", e)
item.update({"product review": "not present"})
try:
item.update({
"recommends":
soup.find('span', attrs={
'class': 'messaging'
}).text.strip()
})
except Exception as e:
item.update({"recommends": "not present"})
try:
item.update({
'shipping': ship.split('\n')
for ship in soup.find('li', attrs={
'class': 'region'
}).text.split('\n\n') if 'Shipping' in ship
})
except Exception as e:
item.update({'shipping': "not present"})
finally:
return item
def stop():
session.close()
if __name__ == "__main__":
product_page_link = output_config.get('product_link')
print(start(product_page_link))
def delete_old_results():
results_folder = output_config.get('results')
for file_old in os.listdir(results_folder):
files = os.path.join(results_folder, file_old)
print("Deleted:{}".format(files))
os.remove(files)
import json
import io
from resources.results import output_config
try:
to_unicode = unicode
except NameError:
to_unicode = str
json_filename = output_config.get('json_file')
class JsonWriter:
def __init__(self):
self.file_json = json_filename
# self.first_line()
def first_line(self):
with open(self.file_json, 'a+') as file_writer:
file_writer.write('[')
def json_writer(self, dict_value):
with io.open(self.file_json, 'a+', encoding='utf8') as outfile:
str_ = json.dumps(dict_value,
indent=4,
sort_keys=True,
separators=(',', ': '),
ensure_ascii=False)
outfile.write(to_unicode(str_))
outfile.write(",\n")
return True
def jsontocsv(json_report):
try:
df = pd.read_json(json_report, orient='columns')
df.to_csv(output_config.get('csv_file'))
except ValueError as ve:
print("Error: Json file not proper.\nplease check in reports folder")
import pandas as pd
from resources.results import output_config
import os
json_report = output_config.get('jsonreport')
def first_modify(file_json):
with open(file_json, 'a+') as file_writer:
file_writer.seek(0, os.SEEK_END) # seek to end of file; f.seek(0, 2) is legal
file_writer.seek(file_writer.tell() - 3, os.SEEK_SET) # go backwards 3 bytes
file_writer.truncate()
file_writer.write("]")
def rewrite(json_file):
with open(json_file,errors='ignore',encoding="utf8") as read_json:
with open(json_report,'a+',errors='ignore',encoding="utf8") as json_writer:
json_writer.write("[\n")
for line in read_json.readlines():
json_writer.write(line)
def jsontocsv(json_report):
try:
df = pd.read_json(json_report, orient='columns')
df.to_csv(output_config.get('csv_file'))
except ValueError as ve:
print("Error: Json file not proper.\nplease check in reports folder")
class NextVariables:
next_filename = output_config.get('pages')
product_filename = output_config.get('product')