def start_paging(self, page_link, tag_id):
p = pagination()
ui = uimanagerclass()
ui.startui()
page_list = p.startPaging(page_link, tag_id)
ui.startpagingui(len(page_list))
return page_list
def start_paging_and_get_all_items(self, page_link, tag_id, item_id):
p = pagination()
ui = uimanagerclass()
ui.startui()
all_items = p.pageingAndProducts(page_link, tag_id, item_id)
all_items_flat_list = [
item for sublist in all_items for item in sublist
]
ui.startextractingproductsurlui(len(all_items_flat_list))
return all_items_flat_list
def func1(self, array, dataset_id, tagList, methodType):
reult_set_list_one = []
thread = threadone(target=startScrapingInThreads,
args=(
array,
dataset_id,
tagList,
methodType,
))
thread.start()
reult_set_list_one.append(thread.join())
print "new Agent started "
db = database()
db.json(reult_set_list_one)
ui = uimanagerclass()
ui.scrapingstartedui(len(array))
def promptmachine(self,url): #prompting machine performances to create relevant threads
memory_status = psutil.virtual_memory()
free_memory = memory_status[4]
util = utils()
start = clock()
process = psutil.Process(util.selenium_test_run(url)) #run selenium to test the consumptions
one_page_time = (clock()-start)
a = process.memory_info().rss
l = process.io_counters()[2]
thredsCount = self.decidethreads(a, l,free_memory)
if(thredsCount == 0):
thredsCount = 1
ui = uimanagerclass()
ui.startmonitorizingui(a,l,free_memory,one_page_time,thredsCount)
return thredsCount
def write_to_csv(self, table_name, db_name):
db = database()
db.toCsv(table_name, db_name)
ui = uimanagerclass()
ui.finishingui(len(0), "CSV FILE CREATED")
def create_mysql(self, jsonFile, db_name, table_name):
db = database()
db.sql(jsonFile, db_name, table_name)
ui = uimanagerclass()
ui.finishingui(len(jsonFile), "SQL TABLE CREATED")
def get_json_obj(self):
db = jsonObjects()
object = db.get_json_from_file()
ui = uimanagerclass()
ui.finishingui(len(object), "JSON")
return object
def get_items_in_pages(self, page_list, tag_id):
p = pagination()
product_list = p.getItemPages(page_list, tag_id)
ui = uimanagerclass()
ui.startextractingproductsurlui(len(product_list))
return product_list