def run(self): try: fact = DataStore() fact.set_ds_type(self.f_type) ds = fact.create_data_store() if self.f_type == "json" or self.f_type == "csv": ds.set_file(self.params[self.f_type]) if self.f_type == "postgre" or self.f_type == "mongo": ds.set_user(self.params["user"]) ds.set_password(self.params["password"]) ds.set_host(self.params["host"]) #ds.set_port(self.params["port"])//use default system port ds.set_db(self.params["db_name"]) ds.set_table(self.params["table"]) ds.connect() for row in self.data: ds.insert_unique(*row) self.sleep(2) except Exception as err: print(err) finally: self.t_finish.emit()
def run(self): try: fact = DataStore() fact.set_ds_type(self.ds_type) ds = fact.create_data_store() ds.set_user(self.params["user"]) ds.set_password(self.params["password"]) ds.set_host(self.params["host"]) #ds.set_port(self.params["port"])//use default system port ds.set_db(self.params["db_name"]) ds.set_table(self.params["table"]) ds.connect() data = ds.get_all_data() self.fill_table.emit(data) self.sleep(2) except Exception as err: print(err) finally: self.t_finish.emit()
"{0}".format(response.status)) if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) logging.debug("[+]App started...") # Starting scrapp posts parser = Scraper(1) data = parser.start() # Create DS, watch DS list... data_store = DataStore() data_store.get_ds_list() # Set DS type as json file and write first 21 records. data_store.set_ds_type("json") ds = data_store.create_data_store() ds.connect() for row in data[:21]: ds.insert_unique(*row) #Set DS type as csv file and write all remaining records. data_store.set_ds_type("csv") ds = data_store.create_data_store() ds.connect() for row in data[22:]: ds.insert_unique(*row) logging.debug("[+]App set_enable !!!")