def create_csv(data_class_list: List[Any], location: str) -> None:
for data in data_class_list:
if not isinstance(data, type(data_class_list[0])):
raise Exception("Different objects types in data class list")
with open(location, "w", newline="\n") as f:
w = DataclassWriter(f, data_class_list, type(data_class_list[0]))
w.write()
def save_epoch_data(self):
file_output = f'./{self.config.output_location}'
self.ensure_path_exists(file_output)
for key in self.client_data:
filename = f'{file_output}/{key}_epochs.csv'
logging.info(f'Saving data at {filename}')
with open(filename, "w") as f:
w = DataclassWriter(f, self.client_data[key], EpochData)
w.write()
def test_wrong_type_items(tmpdir_factory):
tempfile = tmpdir_factory.mktemp("data").join("user_001.csv")
users = [User(name="test", age=40)]
with tempfile.open("w") as f:
with pytest.raises(TypeError):
w = DataclassWriter(f, users, SimpleUser)
w.write()
def start(self) -> object:
myURL = self.URL
sDriver = GetDriver_Selenium()
sDriver.getdriver(myURL)
# print(html)
soup = Mybs4()
while True:
itemDatas = []
html = sDriver.get_PageSources()
soup.set_html(html)
box = soup.selectCSS('div[class="dui-card searchresultitem"]')
#soup = BeautifulSoup ( html, "html.parser" )
#box = soup.select ( 'div[class="dui-card searchresultitem"]' )
for elem in box:
#itemtitle
itemData = ItemData(
soup.select_one_text(
elem, "div[class='content title']>h2"), #itemtitle
soup.select_one_getAttribute(elem, "a", 'href'), #url
soup.select_one_text(
elem,
"div[class='content description price']"), #price
soup.select_one_text(
elem, 'a[class="dui-rating-filter _link"]') # rate
)
itemDatas.append(itemData)
"""
itemtitle=soup.select_one_text(elem, "div[class='content title']>h2")
print(itemtitle)
#url
itemUrl=soup.select_one_getAttribute(elem,"a",'href')
print(itemUrl)
#price
itemPrice=soup.select_one_text(elem,"div[class='content description price']")
print(itemPrice)
#itemRate
itemRate=soup.select_one_text(elem,'a[class="dui-rating-filter _link"]')
print(itemRate)
"""
#print(itemDatas)
with open(f"./rakuten_item_{self.searchWord}.csv", "a") as f:
w = DataclassWriter(f, itemDatas, ItemData)
w.write()
if sDriver.clickxpath("//a[@class='item -next nextPage']"):
pass
else:
break
sDriver.QuitDriver()
def test_create_csv_file(tmpdir_factory):
tempfile = tmpdir_factory.mktemp("data").join("user_001.csv")
users = [User(name="test", age=40)]
with tempfile.open("w") as f:
w = DataclassWriter(f, users, User)
w.write()
with tempfile.open() as f:
reader = DataclassReader(f, User)
saved_users = list(reader)
assert len(saved_users) > 0
assert saved_users[0].name == users[0].name
def outPutCSV_dataclassCSV(self, dataclass, writeMode, dataclasslist=[]):
"""
DataClassWriterを使用してDataClassをcsv出力：need to install dataclass_csv->https://pypi.org/project/dataclass-csv/
:param writeMode:
:param dataclasslist:データ入力済みのデータクラスリスト
:param dataclass:書き込むdataclass型
:return:
"""
if not isinstance(dataclasslist, list):
eMsg = "dataclassList is not List" \
"processing is interrupted"
raise Exception(eMsg)
return
with open(self.outPutPath, writeMode) as f:
w = DataclassWriter(f, dataclasslist, dataclass)
w.write()
def create_final_csv(list):
with open('final.csv', 'w') as new_file:
csv_writer = DataclassWriter(new_file, list, profile)
csv_writer.write()