def start():
globalData = GlobalData.get_instance()
sources = globalData.get_geocatalog_sources()
for source in sources:
e = ExtractionGeocatalogue(source)
listRecords = e.extract_generator_data()
v = VerificationMetadata(source.num_idg)
insert = Insertion(source)
for records in listRecords:
v.check(records)
insert.save_in_db(records)
def merge(UnsortedArray):
ArrayLength = len(UnsortedArray)
if (ArrayLength % 2) == 0:
m = int(ArrayLength / 2)
n1 = int(m)
n2 = int(ArrayLength - m)
else:
m = int((ArrayLength + 1) / 2)
n1 = int(m)
n2 = int(ArrayLength - m)
# create temp arrays
# Left_array = [0] * (n1)
# Right_array = [0] * (n2)
Left_array = [0] * (n1 + 1)
Right_array = [0] * (n2 + 1)
# Copy data to temp arrays Left_array[] and Right_array[]
for i in range(0, n1):
Left_array[i] = (int(UnsortedArray[i]))
Left_array[n1] = 999999999999999999999999999999
for j in range(0, n2):
Right_array[j] = (int(UnsortedArray[m + j]))
Right_array[n2] = 999999999999999999999999999999
# Sort temp arrays with insertion sorting algorithm
Left_arraySorted = Insertion.sorting(Left_array)
Right_arraySorted = Insertion.sorting(Right_array)
# Merge the temp arrays back into sequence[l..r]
i = 0 # Initial index of first subarray
j = 0 # Initial index of second subarray
k = 0 # l # Initial index of merged subarray
for k in range(0, ArrayLength):
if Left_arraySorted[i] <= Right_arraySorted[j]:
UnsortedArray[k] = int(Left_arraySorted[i])
if i == n1:
continue
i += 1
else:
UnsortedArray[k] = int(Right_arraySorted[j])
# if j<n2-1:
if j == n2:
continue
j += 1
return UnsortedArray
def tutorDetailsInsertion(driver):
tutor_name=''
tutor=driver.find_elements_by_xpath('//*[@id="udemy"]/div[2]/div[3]/div[1]/div[3]/div/div/div[4]/div/div[1]/div[3]/div/div/div/div/span/a[1]/span')
for item in tutor:
tutor_name=item.text
tutor_id=db.insertTutorData(tutor_name)
return tutor_id[0]
def InsertionSortFunction(self):
InsertionInitialTime = time.time()
for i in range(0, 20):
UnsortedArrayForInsertion = self.UnsortedArray.copy()
SortedArrayInsertion = Insertion.sorting(UnsortedArrayForInsertion)
self.labelSortedArray.setText(str(SortedArrayInsertion))
InsertionFinalTime = time.time()
InsertionTotalTime = (InsertionFinalTime - InsertionInitialTime) / 20
self.labelinsertionTime.setText(str(InsertionTotalTime))
def courseContentFindingInsertion(driver,course_id):
course_content_list=[]
resource_count='1'
lecture_count=driver.find_elements_by_xpath('//*[@id="udemy"]/div[2]/div[3]/div[1]/div[4]/div[4]/div/div/div/div/div[1]/div/span')
for item in lecture_count:
course_content_list=list((item.text).split("•"))
content=driver.find_elements_by_xpath('//*[@id="udemy"]/div[2]/div[3]/div[1]/div[2]/div/div/div/div[1]/div/div[1]/div[2]/div/div[3]/ul/li[2]/div/span/span')
for item in content:
course_content_list.append(item.text)
print(course_content_list)
duration_detail=course_content_list[2].split(" ")
duration=float(duration_detail[1].rstrip('h')+'.'+duration_detail[2].rstrip('m'))
lecture_count_str=course_content_list[1].split(" ")
lectures=lecture_count_str[1]
section_count=course_content_list[0].split(" ")
sections=section_count[0]
print(course_id,duration,resource_count,lectures,sections)
db.insertCourseContent(course_id,duration,resource_count,lectures,sections)
def main():
driver = webdriver.Chrome(r"lib\chromedriver.exe")
driver.maximize_window()
driver.get("https://www.udemy.com/")
course_file=db.cv_course_group()
for course_name_list in course_file:
course_name=course_name_list[1]
course_group_id=course_name_list[0]
search_keywords=course_name_list[2]
course(driver,course_name,1,course_group_id,search_keywords)
driver.quit()
def createArrayFunc(self):
"""self.UnsortedArray = numpy.random.randint(int(self.spinBoxMin_2.value()), int(self.spinBoxMax_2.value()), int(self.spinBoxLength_2.value()))
self.labelUnsortedArray_2.setText(str(self.UnsortedArray_2))"""
minVal = int(self.spinBoxMin_2.value())
maxVal = int(self.spinBoxMax_2.value())
arrayLength = int(self.spinBoxLength_2.value())
different = maxVal - minVal
if self.checkBoxNoRepeate_2.isChecked():
if different > arrayLength:
def unique_rand(initial, limit, total):
data = []
i = 0
while i < total:
number = numpy.random.randint(initial, limit)
if number not in data:
data.append(number)
i += 1
return data
self.UnsortedArray_2 = unique_rand(minVal, maxVal, arrayLength)
else:
err = "Array length must be greater than difference between limits"
self.labelUnsortedArray_2.setText(str(err))
else:
self.UnsortedArray_2 = numpy.random.randint(
minVal, maxVal, arrayLength)
self.labelUnsortedArray_2.setText(str(self.UnsortedArray_2))
self.SortedArray_2 = Insertion.sorting(self.UnsortedArray_2)
self.labelSortedArray_2.setText(str(self.SortedArray_2))
binaryMax = int(self.spinBoxMax_2.value())
binaryMin = int(self.spinBoxMin_2.value())
self.spinBoxBinary.setMaximum(binaryMax)
self.spinBoxBinary.setMinimum(binaryMin)
self.spinBoxIthSmallest.setMaximum(arrayLength + 1)
import Insertion DaftarAngka = [23,7,32,99,4,15,11,20] Insertion.InsertionSort(DaftarAngka) print(DaftarAngka)
order_products_attributes,
primary_key='product_id,session_id',
foreign_key={
'product_id': 'products(_id)',
'session_id': 'sessions(_id)'
})
print('\n')
# Insertion.insertion_products(conn, data_list_products)
# Insertion.insertion_visitors(conn, data_list_visitors)
# Insertion.insertion_sessions(conn, data_list_sessions)
Insertion.newInsertion_products(conn,
coll1='products',
coll2='stock',
data_input=data_list_products,
attributes=products_attributes,
attributes2=stock_attributes,
fk='_id')
Insertion.newInsertion_visitors(conn,
coll1='visitors',
coll2='visitors_buids',
coll3='recommendation',
coll4='similar_product',
data_input=data_list_visitors,
attributes=visitors_attributes,
attributes2=visitors_buids_attributes,
attributes3=recommendation_attributes,
attributes4=similar_product_attributes,
fk='_id')
Insertion.newInsertion_sessions(conn,
def courseDetails(driver,course_list,course_list_size,count,course_name,course_group_id,search_keywords):
course_link=course_list[count].find_element_by_tag_name('a')
course_specific_url=course_link.get_attribute('href')
driver.get(course_specific_url)
time.sleep(10)
course_details_list=[]
course_heading=driver.find_elements_by_xpath('//*[@id="udemy"]/div[2]/div[3]/div[1]/div[3]/div/div/div[4]/div/div[1]/div[1]/h1')
for item in course_heading:
course_details_list.append(item.text)
course_description=driver.find_elements_by_xpath('//*[@id="udemy"]/div[2]/div[3]/div[1]/div[3]/div/div/div[4]/div/div[1]/div[1]/div')
for item in course_description:
course_details_list.append(item.text)
rating_details=driver.find_elements_by_xpath('//*[@id="udemy"]/div[2]/div[3]/div[1]/div[3]/div/div/div[4]/div/div[1]/div[2]/div[2]/div[1]/div/div/div')
for item in rating_details:
rating_count_data=(item.text).splitlines()
for rating_count_item in rating_count_data:
course_details_list.append(rating_count_item)
rating_count_details=[]
rating_count_details=(course_details_list[4].split(" "))
rating_count=rating_count_details[0].strip('()')
rating_count=rating_count.replace(',','')
students_enrolled=driver.find_elements_by_xpath('//*[@id="udemy"]/div[2]/div[3]/div[1]/div[3]/div/div/div[4]/div/div[1]/div[2]/div[2]/div[2]/div')
for item in students_enrolled:
course_details_list.append(item.text)
student_count_str=course_details_list[5].split(" ")
student_count=student_count_str[0].replace(',','')
last_update=driver.find_elements_by_xpath('//*[@id="udemy"]/div[2]/div[3]/div[1]/div[3]/div/div/div[4]/div/div[1]/div[4]/div[1]/div/div/span[2]')
for item in last_update:
course_details_list.append(item.text)
last_update_str=course_details_list[6].split(" ")
last_updated='01-'+last_update_str[2].replace("/","-")
todayDate=datetime.datetime.strptime(last_updated, "%d-%m-%Y").strftime('%Y-%m-%d')
last_updated=todayDate
lang=''
language=driver.find_elements_by_xpath('//*[@id="udemy"]/div[2]/div[3]/div[1]/div[3]/div/div/div[4]/div/div[1]/div[4]/div[3]/div/div/div/span[1]')
for item in language:
lang=item.text
course_price=driver.find_elements_by_xpath('//*[@id="udemy"]/div[2]/div[3]/div[1]/div[2]/div/div/div/div[1]/div/div[1]/div[2]/div/div[1]/div/div[2]/div/div/div[1]/span[2]/span')
for item in course_price:
course_details_list.append(item.text)
accessibility=driver.find_elements_by_xpath('//*[@id="udemy"]/div[2]/div[3]/div[1]/div[2]/div/div/div/div[1]/div/div[1]/div[2]/div/div[3]/ul/li[3]/div/span/span')
for item in accessibility:
course_details_list.append(item.text)
device=driver.find_elements_by_xpath('//*[@id="udemy"]/div[2]/div[3]/div[1]/div[2]/div/div/div/div[1]/div/div[1]/div[2]/div/div[3]/ul/li[4]/div/span/span')
for item in device:
course_details_list.append(item.text)
certification=driver.find_elements_by_xpath('//*[@id="udemy"]/div[2]/div[3]/div[1]/div[2]/div/div/div/div[1]/div/div[1]/div[2]/div/div[3]/ul/li[5]/div/span/span')
for item in device:
course_details_list.append(item.text)
print(course_details_list)
date = today.strftime("%d-%m-%Y")
date=datetime.datetime.strptime(date, "%d-%m-%Y").strftime('%Y-%m-%d')
name=course_details_list[0]
description=course_details_list[1]
rating=course_details_list[3]
price=course_details_list[7].replace("₹","")
accessibility=course_details_list[8]
devices=course_details_list[9]
certification=1
tutor_id=tutorDetailsInsertion(driver)
print(course_group_id,name,search_keywords,description,'udemy',course_specific_url,tutor_id,rating,rating_count,student_count,last_updated,lang,price,count,accessibility,devices,certification,date,date)
course_id_list=db.insertCourseData(course_group_id,name,search_keywords,description,'udemy',course_specific_url,tutor_id,rating,rating_count,student_count,last_updated,"ENG",price,count,accessibility,devices,certification,date,date)
course_id=int(course_id_list[0])
courseContentFindingInsertion(driver,course_id)
count+=1
course(driver,course_name,count,course_group_id,search_keywords)
def CompareFunc(self):
self.graphicsView.clear()
self.StepsCounter = int(self.spinBoxLengthSteps.value())
InsertionTimes = []
MergeTimes = []
QuickTimes = []
RandQuickTimes = []
BubbleTimes = []
CountingTimes = []
HeapTimes = []
RadixTimes = []
x = []
x.append(self.StepsCounter)
while self.StepsCounter < (int(self.spinBoxMaxLength.value()) + 1):
QApplication.processEvents()
self.CreateRandomArrayFunctionForComparison()
if self.checkBoxInsertion.isChecked():
InsertionInitial = time.time()
for i in range(0, 10):
UnsortedArrayForInsertion = self.UnsortedArrayComparison.copy(
)
QApplication.processEvents()
Insertion.sorting(UnsortedArrayForInsertion)
InsertionFinal = time.time()
InsertionTotal = (InsertionFinal - InsertionInitial) / 10
InsertionTimes.append(InsertionTotal)
else:
InsertionTimes = []
if self.checkBoxBubble.isChecked():
BubbleInitial = time.time()
for i in range(0, 10):
UnsortedArrayBubble = self.UnsortedArrayComparison.copy()
QApplication.processEvents()
Bubble.Sorting(UnsortedArrayBubble)
BubbleFinal = time.time()
BubbleTotal = (BubbleFinal - BubbleInitial) / 10
BubbleTimes.append(BubbleTotal)
else:
BubbleTimes = []
if self.checkBoxCounting.isChecked():
CountingInitial = time.time()
for i in range(0, 10):
UnsortedArrayCounting = self.UnsortedArrayComparison.copy()
maxRange = int(self.spinBoxMaxLength.value()) * 2
QApplication.processEvents()
countingSort.countSort(UnsortedArrayCounting, maxRange)
CountingFinal = time.time()
CountingTotal = (CountingFinal - CountingInitial) / 10
CountingTimes.append(CountingTotal)
else:
CountingTimes = []
if self.checkBoxHeap.isChecked():
HeapInitial = time.time()
for i in range(0, 10):
UnsortedArrayHeap = self.UnsortedArrayComparison.copy()
QApplication.processEvents()
heapsort.heapSort(UnsortedArrayHeap)
HeapFinal = time.time()
HeapTotal = (HeapFinal - HeapInitial) / 10
HeapTimes.append(HeapTotal)
else:
HeapTimes = []
if self.checkBoxMerge.isChecked():
MergeInitial = time.time()
for i in range(0, 10):
UnsortedArrayMerge = self.UnsortedArrayComparison.copy()
QApplication.processEvents()
Merge.merge(UnsortedArrayMerge)
MergeFinal = time.time()
MergeTotal = (MergeFinal - MergeInitial) / 10
MergeTimes.append(MergeTotal)
else:
MergeTimes = []
if self.checkBoxQuick.isChecked():
QuickInitial = time.time()
for i in range(0, 10):
UnsortedArrayQuick = self.UnsortedArrayComparison.copy()
QApplication.processEvents()
quicksort.QuickSort(UnsortedArrayQuick, 0,
self.StepsCounter - 1)
QuickFinal = time.time()
QuickTotal = (QuickFinal - QuickInitial) / 10
QuickTimes.append(QuickTotal)
else:
QuickTimes = []
if self.checkBoxRandQuick.isChecked():
RandQuickInitial = time.time()
for i in range(0, 10):
UnsortedArrayQuick = self.UnsortedArrayComparison.copy()
QApplication.processEvents()
quicksort.QuickSort(UnsortedArrayQuick, 0,
self.StepsCounter - 1)
RandQuickFinal = time.time()
RandQuickTotal = (RandQuickFinal - RandQuickInitial) / 10
RandQuickTimes.append(RandQuickTotal)
else:
RandQuickTimes = []
if self.checkBoxRadix.isChecked():
radixInitial = time.time()
for i in range(0, 10):
UnsortedArrayRadix = self.UnsortedArrayComparison.copy()
QApplication.processEvents()
radixSort.radixSort(UnsortedArrayRadix)
radixFinal = time.time()
radixTotal = (radixFinal - radixInitial) / 10
RadixTimes.append(radixTotal)
else:
RadixTimes = []
self.StepsCounter = self.StepsCounter + int(
self.spinBoxLengthSteps.value())
x.append(self.StepsCounter)
self.graphicsView.setTitle("Time Comparison Graph")
self.graphicsView.setLabel('left', 'Time', units='s')
self.graphicsView.setLabel('bottom', 'Array Length')
if self.checkBoxInsertion.isChecked():
self.graphicsView.plot(InsertionTimes,
name="Insertion",
pen=pyqtgraph.mkPen('w', width=1.5))
if self.checkBoxMerge.isChecked():
self.graphicsView.plot(MergeTimes,
name="Merge",
pen=pyqtgraph.mkPen('r', width=1.5))
if self.checkBoxQuick.isChecked():
self.graphicsView.plot(QuickTimes,
name="Quick",
pen=pyqtgraph.mkPen('g', width=1.5))
if self.checkBoxRandQuick.isChecked():
self.graphicsView.plot(RandQuickTimes,
name="RandQuick",
pen=pyqtgraph.mkPen('m', width=1.5))
if self.checkBoxHeap.isChecked():
self.graphicsView.plot(HeapTimes,
name="Heap",
pen=pyqtgraph.mkPen('y', width=1.5))
if self.checkBoxBubble.isChecked():
self.graphicsView.plot(BubbleTimes,
name="Bubble",
pen=pyqtgraph.mkPen('k', width=1.5))
if self.checkBoxCounting.isChecked():
self.graphicsView.plot(CountingTimes,
name='Counting',
pen=pyqtgraph.mkPen('g', width=1.5),
style=Qt.DashLine)
if self.checkBoxRadix.isChecked():
self.graphicsView.plot(RadixTimes,
name='Radix',
pen=pyqtgraph.mkPen('r',
width=1.5,
style=Qt.DotLine))