def run(self):
try:
fact = DataStore()
fact.set_ds_type(self.f_type)
ds = fact.create_data_store()
if self.f_type == "json" or self.f_type == "csv":
ds.set_file(self.params[self.f_type])
if self.f_type == "postgre" or self.f_type == "mongo":
ds.set_user(self.params["user"])
ds.set_password(self.params["password"])
ds.set_host(self.params["host"])
#ds.set_port(self.params["port"])//use default system port
ds.set_db(self.params["db_name"])
ds.set_table(self.params["table"])
ds.connect()
for row in self.data:
ds.insert_unique(*row)
self.sleep(2)
except Exception as err:
print(err)
finally:
self.t_finish.emit()
def run(self):
try:
fact = DataStore()
fact.set_ds_type(self.ds_type)
ds = fact.create_data_store()
ds.set_user(self.params["user"])
ds.set_password(self.params["password"])
ds.set_host(self.params["host"])
#ds.set_port(self.params["port"])//use default system port
ds.set_db(self.params["db_name"])
ds.set_table(self.params["table"])
ds.connect()
data = ds.get_all_data()
self.fill_table.emit(data)
self.sleep(2)
except Exception as err:
print(err)
finally:
self.t_finish.emit()
"{0}".format(response.status))
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
logging.debug("[+]App started...")
# Starting scrapp posts
parser = Scraper(1)
data = parser.start()
# Create DS, watch DS list...
data_store = DataStore()
data_store.get_ds_list()
# Set DS type as json file and write first 21 records.
data_store.set_ds_type("json")
ds = data_store.create_data_store()
ds.connect()
for row in data[:21]:
ds.insert_unique(*row)
#Set DS type as csv file and write all remaining records.
data_store.set_ds_type("csv")
ds = data_store.create_data_store()
ds.connect()
for row in data[22:]:
ds.insert_unique(*row)
logging.debug("[+]App set_enable !!!")
