def start_paging(self, page_link, tag_id): p = pagination() ui = uimanagerclass() ui.startui() page_list = p.startPaging(page_link, tag_id) ui.startpagingui(len(page_list)) return page_list
def start_paging_and_get_all_items(self, page_link, tag_id, item_id): p = pagination() ui = uimanagerclass() ui.startui() all_items = p.pageingAndProducts(page_link, tag_id, item_id) all_items_flat_list = [ item for sublist in all_items for item in sublist ] ui.startextractingproductsurlui(len(all_items_flat_list)) return all_items_flat_list
def func1(self, array, dataset_id, tagList, methodType): reult_set_list_one = [] thread = threadone(target=startScrapingInThreads, args=( array, dataset_id, tagList, methodType, )) thread.start() reult_set_list_one.append(thread.join()) print "new Agent started " db = database() db.json(reult_set_list_one) ui = uimanagerclass() ui.scrapingstartedui(len(array))
def promptmachine(self,url): #prompting machine performances to create relevant threads memory_status = psutil.virtual_memory() free_memory = memory_status[4] util = utils() start = clock() process = psutil.Process(util.selenium_test_run(url)) #run selenium to test the consumptions one_page_time = (clock()-start) a = process.memory_info().rss l = process.io_counters()[2] thredsCount = self.decidethreads(a, l,free_memory) if(thredsCount == 0): thredsCount = 1 ui = uimanagerclass() ui.startmonitorizingui(a,l,free_memory,one_page_time,thredsCount) return thredsCount
def write_to_csv(self, table_name, db_name): db = database() db.toCsv(table_name, db_name) ui = uimanagerclass() ui.finishingui(len(0), "CSV FILE CREATED")
def create_mysql(self, jsonFile, db_name, table_name): db = database() db.sql(jsonFile, db_name, table_name) ui = uimanagerclass() ui.finishingui(len(jsonFile), "SQL TABLE CREATED")
def get_json_obj(self): db = jsonObjects() object = db.get_json_from_file() ui = uimanagerclass() ui.finishingui(len(object), "JSON") return object
def get_items_in_pages(self, page_list, tag_id): p = pagination() product_list = p.getItemPages(page_list, tag_id) ui = uimanagerclass() ui.startextractingproductsurlui(len(product_list)) return product_list