def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'Falmouth University'
try:
location = 'Woodlane, Falmouth'
location = remove_tags(location)
#print(location)
except:
location = 'N/A'
#print(location)
try:
department = response.xpath(
'/html/body/div[1]/div/div/div[2]/div[1]/div[2]/p[2]/a[3]/strong'
).extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
department = department.replace(' ', '')
department = department.replace('\n', '')
department = department.replace('Our Staff', '')
#print(department)
except:
department = ''
#print(department)
try:
degree_name = response.xpath('//h1').extract()[0]
degree_name = remove_tags(degree_name)
degree_name = degree_name.split()[-1]
#degree_name = re.findall('(.*)\n.*',degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
degree_name = degree_name.replace(' ', '')
#print(degree_name)
except:
degree_name = 'N/A'
#print(degree_name)
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath('//h1').extract()[0]
programme_en = remove_tags(programme_en)
#programme_en = re.findall(' (.*)',programme_en)[0]
programme_en = programme_en.replace(degree_name, '')
programme_en = programme_en.replace(' ', '')
#programme_en = programme_en.replace('\n', '')
#programme_en = re.findall((' '),'')[0]
#programme_en = re.findall("(.*)\(.*\)",programme_en)[0]
#programme_en = programme_en.replace('\n','')
#programme_en = programme_en.replace(' ','')
#print(programme_en)
except:
programme_en = 'N/A'
#print(programme_en)
try:
overview_en = response.xpath(
'//div[@class = "field-body"]').extract()[0]
overview_en = remove_tags(overview_en)
overview_en = overview_en.replace(' ', '')
#overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n', '')
overview_en = overview_en.replace('\r\n', '')
overview_en = overview_en.replace('\n', '')
overview_en = '<div>' + overview_en + '</div>'
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = 'N/A'
#print(overview_en)
try:
start_date = '9'
#print(start_date)
except:
start_date = ''
try:
modules_en = response.xpath('//body').extract()[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n', '\n')
modules_en = modules_en.replace('\r\n', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace('\n', '')
modules_en = re.findall('Course outline(.*)Facilities',
modules_en)[0]
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]'
).extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ', '')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath(
'//div[@class = "call-content"]').extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = "<div>" + rntry_requirements_en + "</div>"
rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\n', '')
rntry_requirements_en = rntry_requirements_en.replace(' ', '')
#rntry_requirements_en =rntry_requirements_en.replace(' ','')
#print(rntry_requirements_en)
except:
rntry_requirements_en = 'N/A'
#print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = ''
except:
require_chinese_en = ''
try:
ielts_desc = 'If English is not your first language, you\'ll need to demonstrate English language skills that are sufficiently developed for successful completion of your studies. We accept a range of recognised English language qualifications that are equivalent to the International English Language Testing System (IELTS) Academic minimum score of 6.5 overall, with a minimum of 6.0 in Reading, Writing, Speaking and Listening. International applicants who require a Tier 4 student visa to study in the UK, must take an approved Secure English Language Test (SELT) to fulfil government visa requirements, or have a recognised language test approved and vouched for by the University. Our Admissions team can help with any questions you may have about study visas or suitable language tests.'
#print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
ielts = '6.5'
#ielts =remove_tags(ielts)
#ielts = re.findall('IELTS(.*)',ielts)[0]
#ielts = re.findall('(\d\.\d)',ielts)[0]
#print(ielts)
except:
ielts = 6.5
#print(ielts)
try:
ielts_l = 6.0
#print(ielts_l)
ielts_l = remove_tags(ielts_l)
except:
ielts_l = 6.0
try:
ielts_s = 6.0
except:
ielts_s = 6.0
try:
ielts_r = 6.0
except:
ielts_r = 6.0
try:
ielts_w = 6.0
except:
ielts_w = 6.0
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = 0
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = 0
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = 0
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = 0
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = 0
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = 0
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = 0
try:
interview_desc_en = response.xpath(
'//*[@id="entry-requirements-accordion-0"]/div[1]').extract(
)[0]
interview_desc_en = remove_tags(interview_desc_en)
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
interview_desc_en = "<div>" + interview_desc_en + "</div>"
#print(interview_desc_en)
except:
interview_desc_en = 'N/A'
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath('//body').extract()[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\r\n', '')
career_en = career_en.replace(' ', '')
career_en = career_en.replace('\n', '')
career_en = re.findall(
'Potential careers include:(.*)\(function\(\)', career_en)[0]
career_en = career_en.replace('&', '')
career_en = "<div><span>" + career_en + "</span></div>"
#print(career_en)
except:
career_en = 'N/A'
#print(career_en)
try:
apply_desc_en = '<p>The Global Education Unit and academic staff frequently visit China throughout the year to meet prospective students and work with our academic partners. The university also work with International Student Recruitment Representatives and Recruitment Agents who can assist you in your application process. Alternatively, you can apply to the university directly, where an allocated international officer will help you through the applications process.</p>'
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = '<p>Undergraduate Programmes, applicants must have one of the following: Graduation Certificate from a Specialised College / School (Zhongzhuan) Chinese University / College Entrance Examination (Gaokao) Graduation Certificate (Zhuanke / Dazhuan / Gaozhi) – (may be considered for advanced entry to certain programmes) Successful completion of a recognised Foundation programme Postgraduate Programmes, applicants must have a Bachelor Degree. School certificates and certified transcripts must accompany your application.</p>'
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 0
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = 1
duration = remove_tags(duration)
#duration = remove_tags(duration)
#duration = re.findall('(\d) Years',duration)[0]
# if '36' in duration:
# duration = '3'
# elif '16' in duration:
# duration = '1'
# elif '12' in duration:
# duration = '1'
# elif '3' in duration:
# duration = '3'
# elif '2' in duration:
# duration = '2'
# elif '1' in duration:
# duration = '1'
# elif 'two' in duration:
# duration = '2'
# else:
# duration = '1'
#print(duration)
except:
duration = '0'
#print(duration)
try:
other = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a'
).extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div[1]/table[1]/tbody/tr[11]/td[2]'
).extract()[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div/table[1]'
).extract()[0]
alevel = remove_tags(alevel)
alevel = re.findall("(\w\w\w) at A Level", alevel)[0]
#print(alevel)
except:
alevel = 'CC'
#print(alevel)
try:
ucascode = response.xpath(
'/html/body/div[3]/div[1]/div/div/div[2]/div/div[1]/div[1]/div[2]'
).extract()[0]
ucascode = remove_tags(ucascode)
#print(ucascode)
except:
ucascode = ''
#print(ucascode)
try:
tuition_fee = '15000'
# tuition_fee = remove_tags(tuition_fee)
# tuition_fee = tuition_fee.replace('£','')
# tuition_fee = tuition_fee.replace(',','')
# tuition_fee = tuition_fee.replace('*','')
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\r\n','')
# tuition_fee = tuition_fee.replace('\n','')
#
# tuition_fee = re.findall('(\d\d\d\d\d)',tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#print(tuition_fee)
try:
assessment_en = response.xpath('//body').extract()[0]
assessment_en = remove_tags(assessment_en)
assessment_en = assessment_en.replace(' ', '')
assessment_en = assessment_en.replace('\r\n', '')
assessment_en = assessment_en.replace('\n', '')
assessment_en = re.findall('Assessment(.*)Careers',
assessment_en)[0]
assessment_en = "<div>" + assessment_en + "</div>"
#print(assessment_en)
except:
assessment_en = 'N/A'
#print(assessment_en)
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 2
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 1
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
item["alevel"] = alevel
item["ib"] = ib
item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
item["assessment_en"] = assessment_en
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'University of Birmingham'
try:
location = 'warwick'
#location = remove_tags(location)
#print(location)
except:
location = 'N/A'
#print(location)
try:
department = response.xpath('').extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
department = department.replace(' ', '')
department = department.replace('\n', '')
department = department.replace('Our Staff', '')
#print(department)
except:
department = ''
#print(department)
try:
degree_name = response.xpath(
'//*[@id="main"]/div/div/div/div/div[2]/div[1]/section/h3/strong'
).extract()[0]
degree_name = remove_tags(degree_name)
degree_name = re.findall('.* \((.*)\)', degree_name)[0]
#degree_name = re.findall('(.*)\n.*',degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
degree_name = degree_name.replace(' ', '')
#print(degree_name)
except:
degree_name = 'BA'
#print(degree_name)
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath(
'//*[@id="main"]/div/div/div/div/div[2]/div[1]/section/h3/strong'
).extract()[0]
programme_en = remove_tags(programme_en)
programme_en = programme_en.replace('\r\n', '')
#programme_en = re.findall('',programme_en)[0]
programme_en = programme_en.replace(' ', ' ')
programme_en = programme_en.replace(degree_name, '')
programme_en = programme_en.replace('()', '')
#print(programme_en)
except:
programme_en = 'N/A'
#print(programme_en)
try:
overview_en = response.xpath(
'//*[@id="course-tab-1"]/section/p').extract()[0]
overview_en = remove_tags(overview_en)
overview_en = overview_en.replace(' ', '')
#overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n', '')
overview_en = overview_en.replace('\r\n', '')
overview_en = overview_en.replace('\n', '')
overview_en = '<div>' + overview_en + '</div>'
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = 'N/A'
#print(overview_en)
try:
start_date = response.xpath('//*[@id="course-tab-5"]').extract()[0]
start_date = remove_tags(start_date)
start_date = start_date.replace('\r\n', '')
start_date = start_date.replace(' ', ' ')
start_date = start_date.replace('\n', '')
start_date = re.findall('Start Date(.*)', start_date)[0]
if 'October' in start_date:
start_date = '2019-10'
elif '24' in start_date:
start_date = '2019-9-24'
else:
start_date = '2019-9'
#print(start_date)
except:
start_date = 'N/A'
#print(start_date)
try:
modules_en = response.xpath('//*[@id="course-tab-3"]').extract()[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n', '\n')
modules_en = modules_en.replace('\r\n', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace('\n', '')
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]'
).extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ', '')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath(
'//*[@id="course-tab-2"]').extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = "<div>" + rntry_requirements_en + "</div>"
rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\n', '')
rntry_requirements_en = rntry_requirements_en.replace(' ', '')
#rntry_requirements_en =rntry_requirements_en.replace(' ','')
#print(rntry_requirements_en)
except:
rntry_requirements_en = 'N/A'
#print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = ''
except:
require_chinese_en = ''
try:
ielts_desc = 'https://warwick.ac.uk/study/undergraduate/courses-2019/economicspoliticsinternational'
#print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
#ielts = '6.5'
#ielts =remove_tags(ielts)
#ielts = re.findall('IELTS(.*)',ielts)[0]
ielts = 0
#print(ielts)
except:
ielts = 0
#print(ielts)
try:
#ielts_l = '5.5'
ielts_l = 0
#print(ielts_l)
#ielts_l = remove_tags(ielts_l)
except:
ielts_l = 0
try:
ielts_s = ielts_l
except:
ielts_s = ielts_l
try:
ielts_r = ielts_l
except:
ielts_r = ielts_l
try:
ielts_w = ielts_l
except:
ielts_w = ielts_l
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = 0
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = 0
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = 0
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = 0
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = 0
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = 0
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = 0
try:
interview_desc_en = response.xpath(
'//*[@id="entry-requirements-accordion-0"]/div[1]').extract(
)[0]
interview_desc_en = remove_tags(interview_desc_en)
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
interview_desc_en = "<div>" + interview_desc_en + "</div>"
#print(interview_desc_en)
except:
interview_desc_en = 'N/A'
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath(
'//*[@id="EmployabilityTab"]').extract()[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\r\n', '')
career_en = career_en.replace(' ', ' ')
career_en = career_en.replace('\n', '')
career_en = "<div><span>" + career_en + "</span></div>"
print(career_en)
except:
career_en = ''
print(career_en)
try:
apply_desc_en = '<span>Assessing each application fairly and consistently within an extremely competitive field is a difficult task. It is carried out by course selectors (Admissions Tutors) who are academics in departments and by professionals in the Undergraduate Admissions Team to ensure that decisions are made fairly, taking into account as much information about applicants as possible. Applications are assessed on their own merits and in competition with others, as we receive many more applications for most courses than there are places available. Selectors judge the evidence provided on the UCAS application against the criteria set for the chosen course. They take into account existing academic achievements and the context within which they have been achieved (including any exceptional circumstances), predicted grades, the personal statement and the academic reference. Remember that selectors want to hear about you and your interests and potential – there is no one-size-fits-all approach! As a consequence of the high level of competition for our courses, and because we want to consider your full profile and your potential as an individual rather than simply looking at your actual or predicted grades, it may take some time to communicate a decision to you. We will keep you informed of the status of your application during the admissions process. Successful candidates will receive an offer which the selector feels is most appropriate, though typical offer levels are listed for each course. We will provide feedback to candidates to whom we are not able to make an offer when this is requested in writing. You should be aware that decisions are made on a highly competitive basis and therefore we are often unable to make offers to all applicants who meet, or even exceed, the typical entry requirements.</span>'
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = ''
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 13
dead_time = '1-15'
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = response.xpath('//*[@id="course-tab-5"]').extract()[0]
#duration = remove_tags(duration)
duration = remove_tags(duration)
duration = duration.replace(' ', '')
duration = duration.replace('\r\n', '')
duration = duration.replace('\n', '')
duration = re.findall('Duration(.*)', duration)[0]
#duration = re.findall('(\d) Years',duration)[0]
if '4' in duration:
duration = '4'
elif '3' in duration:
duration = '3'
elif '5' in duration:
duration = '5'
elif '2' in duration:
duration = '2'
elif '1' in duration:
duration = '1'
elif 'two' in duration:
duration = '2'
else:
duration = 'N/A'
#print(duration)
except:
duration = 'N/A'
#print(duration)
try:
other = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a'
).extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath(
'//*[@id="main"]/div/div/div/div/div[2]/div[1]/section/p[1]/strong[2]'
).extract()[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath(
'//*[@id="main"]/div/div/div/div/div[2]/div[1]/section/p[1]/strong[1]'
).extract()[0]
alevel = remove_tags(alevel)
alevel = re.findall('entry, (.*), IB', alevel)[0]
#alevel = alevel.replace('*','')
#alevel = re.findall("(\w\w\w)",alevel)[0]
#print(alevel)
except:
alevel = 'N/A'
#print(alevel)
try:
ucascode = response.xpath('//*[@id="course-tab-5"]').extract()[0]
ucascode = remove_tags(ucascode)
ucascode = ucascode.replace('\r\n', '')
ucascode = ucascode.replace('\n', '')
ucascode = ucascode.replace(' ', ' ')
ucascode = re.findall('UCAS Code(.*)Award', ucascode)[0]
ucascode = ucascode.replace(' ', '')
ucascode = ucascode.replace(' ', '')
#print(ucascode)
except:
ucascode = 'N/A'
#print(ucascode)
try:
tuition_fee = response.xpath(
'/html/body/div[1]/div/div[1]/section[7]/div/div/div[3]/div/div[2]'
).extract()[0]
tuition_fee = remove_tags(tuition_fee)
tuition_fee = tuition_fee.replace('£', '')
tuition_fee = tuition_fee.replace(',', '')
tuition_fee = tuition_fee.replace('*', '')
tuition_fee = tuition_fee.replace(' ', '')
tuition_fee = tuition_fee.replace('\r\n', '')
tuition_fee = tuition_fee.replace('\n', '')
tuition_fee = re.findall('(\d\d\d\d\d)', tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#print(tuition_fee)
try:
assessment_en = response.xpath(
'//*[@id="assessment-methods"]/div/p').extract()[0]
assessment_en = remove_tags(assessment_en)
assessment_en = assessment_en.replace('\r\n', '')
assessment_en = assessment_en.replace(' ', '')
assessment_en = assessment_en.replace('\n', '')
assessment_en = "<div><span>" + assessment_en + "</span></div>"
#print(assessment_en)
except:
assessment_en = ''
#print(assessment_en)
application_open_date = '2018-10-6/2018-10-20'
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 1
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 4
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
item["alevel"] = alevel
item["ib"] = ib
item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
item["assessment_en"] = assessment_en
item["application_open_date"] = application_open_date
#item["apply_pre"] = ''
yield item
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'University of Westminster'
try:
location = response.xpath('//*[@id="location"]/a').extract()[0]
location = remove_tags(location)
#print(location)
except:
location = 'N/A'
#print(location)
try:
department = response.xpath(
'//ul[@class="course-department"]').extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
department = department.replace(' ', '')
department = department.replace('\n', '')
department = department.replace('Our Staff', '')
#print(department)
except:
department = ''
#print(department)
try:
degree_name = response.xpath(
'/html/body/div[2]/div[2]/div/header/h1/span').extract()[0]
degree_name = remove_tags(degree_name)
#degree_name = degree_name.split()[0]
#degree_name = re.findall('(.*)\n.*',degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
degree_name = degree_name.replace(' ', '')
#print(degree_name)
except:
degree_name = 'N/A'
#print(degree_name)
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath(
'/html/body/div[2]/div[2]/div/header/h1/text()').extract()[0]
#programme_en = remove_tags(programme_en)
#programme_en = re.findall(' (.*)',programme_en)[0]
programme_en = programme_en.replace(degree_name, '')
programme_en = programme_en.replace(' ', '')
#programme_en = programme_en.replace('\n', '')
#programme_en = re.findall((' '),'')[0]
#programme_en = re.findall("(.*)\(.*\)",programme_en)[0]
#programme_en = programme_en.replace('\n','')
#programme_en = programme_en.replace(' ','')
#print(programme_en)
except:
programme_en = 'N/A'
#print(programme_en)
try:
overview_en = response.xpath(
'/html/body/div[4]/div/section/div[2]/div[4]/div[1]').extract(
)[0]
overview_en = remove_tags(overview_en)
overview_en = overview_en.replace(' ', '')
#overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n', '')
overview_en = overview_en.replace('\r\n', '')
overview_en = overview_en.replace('\n', '')
overview_en = '<div>' + overview_en + '</div>'
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = 'N/A'
#print(overview_en)
try:
start_date = '9'
#print(start_date)
except:
start_date = ''
try:
modules_en = response.xpath(
'/html/body/div[4]/div/section/div[2]/div[4]/div[2]').extract(
)[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n', '\n')
modules_en = modules_en.replace('\r\n', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace('\n', '')
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]'
).extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ', '')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath(
'//div[@class = "layout__1col"]').extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = "<div>" + rntry_requirements_en + "</div>"
rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\n', '')
rntry_requirements_en = rntry_requirements_en.replace(' ', '')
#rntry_requirements_en =rntry_requirements_en.replace(' ','')
#print(rntry_requirements_en)
except:
rntry_requirements_en = 'N/A'
#print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = '<div>Applying for a course If you are interested in studying for a Master\'s degree (eg MA, MSc or LLM) or research programme (eg MPhil or PhD) at the University of Westminster, you will need to check that you meet our entry requirements. We have included typical entry requirements below, but these are simply a guideline. Some courses may also have specific subject or grade requirements, which will be listed on individual course pages. For entry to a Master\'s degree that requires a UK 2:2 degree, you will typically need to have one of the following: Bachelor degree from 211, 985 or top national universities with an overall average grade of 70% Bachelor degree from national universities with an overall average grade of 75% Bachelor degree from high-ranking private universities with an overall average grade of 75% Master degrees with an overall average grade of 60% For entry to a Master\'s degree that requires a UK 2:1 degree, you will typically need to have one of the following: Bachelor degree from 211, 985 or top national universities with an overall average grade of 75% Bachelor degree from national universities with an overall average grade of 80% Bachelor degree from high-ranking private universities with an overall average grade of 80% Master degrees with an overall average grade of 70% For more details on the application process, visit our How to apply page. To search or browse our postgraduate courses, visit our Postgraduate page. If your qualifications are not listed If your qualifications are not listed above, you may still be able to apply for entry to our courses – just get in touch with our overseas representatives or course enquiries team via the contact details on this page. If you find that your qualifications do not meet our entry requirements, you may want to consider applying for one of the University preparation courses offered by our partner college – see below for details. University preparation courses Our partner college, Kaplan International College London, offers a Pre-Master\'s course leading to entry to a number of our Master\'s degrees. To find out more and apply, visit the Kaplan International College London website.</div>'
except:
require_chinese_en = ''
try:
ielts_desc = ''
#print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
ielts = response.xpath('//body').extract()[0]
ielts = remove_tags(ielts)
ielts = ielts.replace('\r\n', '')
ielts = ielts.replace('\n', '')
#ielts = re.findall('International Baccalaureate(.*)',ielts)[0]
ielts = re.findall('IELTS(.*)', ielts)[0]
ielts = re.findall('(\d\.\d)', ielts)[0]
#print(ielts)
except:
ielts = '0'
#print(ielts)
try:
ielts_l = response.xpath('//body').extract()[0]
ielts_l = remove_tags(ielts_l)
ielts_l = re.findall('IELTS(.*)', ielts_l)[0]
ielts_l = re.findall('(\d\.\d)', ielts_l)[1]
print(ielts_l)
#ielts_l = remove_tags(ielts_l)
except:
ielts_l = 0
try:
ielts_s = ielts_l
except:
ielts_s = 0
try:
ielts_r = ielts_l
except:
ielts_r = 0
try:
ielts_w = ielts_l
except:
ielts_w = 0
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = 0
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = 0
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = 0
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = 0
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = 0
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = 0
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = 0
try:
interview_desc_en = response.xpath(
'//*[@id="entry-requirements-accordion-0"]/div[1]').extract(
)[0]
interview_desc_en = remove_tags(interview_desc_en)
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
interview_desc_en = "<div>" + interview_desc_en + "</div>"
#print(interview_desc_en)
except:
interview_desc_en = 'N/A'
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath(
'/html/body/div[4]/div/section/div[2]/div[14]/div').extract(
)[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\r\n', '')
career_en = career_en.replace(' ', '')
career_en = career_en.replace('\n', '')
career_en = "<div><span>" + career_en + "</span></div>"
#print(career_en)
except:
career_en = ''
#print(career_en)
try:
apply_desc_en = '<p>To apply for a postgraduate course, simply click on the red \'Apply\' button at the top of our course pages. You will be directed to the UCAS Postgraduate website to make your application. When completing the application form on UCAS Postgraduate remember to specify: why you wish to enrol on the course any relevant experience you have why you think you should be given a place</p>'
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = '<span>why you wish to enrol on the course any relevant experience you have why you think you should be given a place<\span>'
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 0
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = response.xpath(
'/html/body/div[4]/div/section/div[2]/div[2]/div/div[2]/div[2]/div/span[2]'
).extract()[0]
duration = remove_tags(duration)
#duration = remove_tags(duration)
#duration = re.findall('(\d) Years',duration)[0]
if '6' in duration:
duration = '6'
elif '5' in duration:
duration = '5'
elif '4' in duration:
duration = '4'
elif '3' in duration:
duration = '3'
elif '2' in duration:
duration = '2'
elif '1' in duration:
duration = '1'
#print(duration)
except:
duration = '0'
#print(duration)
try:
other = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a'
).extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath(
'//*[@id]/div/div/div/ul[1]/li[2]').extract()[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath(
'//*[@id]/div/div/div/ul[1]/li[1]').extract()[0]
alevel = remove_tags(alevel)
#alevel = re.findall("(\w\w\w) at A Level",alevel)[0]
#print(alevel)
except:
alevel = 'N/A'
#print(alevel)
try:
ucascode = response.xpath(
'//*[@id="js-anchor-nav"]/div/div/div/div/p/strong[1]'
).extract()[0]
ucascode = remove_tags(ucascode)
#print(ucascode)
except:
ucascode = ''
#print(ucascode)
try:
tuition_fee = response.xpath(
'/html/body/div[4]/div/section/div[2]/div[2]/div/div[2]/div[1]/div[2]/span[2]/a'
).extract()[0]
tuition_fee = remove_tags(tuition_fee)
tuition_fee = tuition_fee.replace('£', '')
tuition_fee = tuition_fee.replace(',', '')
tuition_fee = tuition_fee.replace('*', '')
tuition_fee = tuition_fee.replace(' ', '')
tuition_fee = tuition_fee.replace('\r\n', '')
tuition_fee = tuition_fee.replace('\n', '')
tuition_fee = re.findall('(\d\d\d\d\d)', tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#print(tuition_fee)
assessment_en = ''
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 1
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 3
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
item["alevel"] = alevel
item["ib"] = ib
item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
item["assessment_en"] = assessment_en
#item["apply_pre"] = ''
yield item
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'Birkbeck, University of London'
try:
location = 'Central London'
#location = remove_tags(location)
#location = remove_tags(location)
#print(location)
except:
location = 'n/a'
#print(location)
try:
department = response.xpath('').extract()
department = remove_tags(department)
except:
department = ''
try:
degree_name = response.xpath('//h1').extract()[0]
degree_name = remove_tags(degree_name)
degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
#print(degree_name)
except:
degree_name = ''
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath('//h1').extract()[0]
programme_en = remove_tags(programme_en)
programme_en = re.findall("(.*)\(.*\)",programme_en)[0]
#programme_en = programme_en.replace('\n','')
#programme_en = programme_en.replace(' ','')
#print(programme_en)
except:
programme_en = ''
#print(programme_en)
try:
overview_en = response.xpath('//*[@id="content"]/div[5]').extract()[0]
overview_en = remove_tags(overview_en)
overview_en = '<div>'+overview_en +'</div>'
overview_en = overview_en.replace(' ','')
overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n','\n')
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = ''
try:
start_date = '10'
#print(start_date)
except:
start_date = ''
try:
modules_en = response.xpath('//*[@id="courseStructure"]').extract()[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n','\n')
modules_en = modules_en.replace('\r\n','')
modules_en = modules_en.replace(' ','')
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
apply_fee = 0
try:
degree_requirements = response.xpath('//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]').extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ','')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath('//*[@id="content"]/div[8]/div/div[2]').extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = rntry_requirements_en.replace(' ','')
rntry_requirements_en = rntry_requirements_en.replace('\n','')
rntry_requirements_en = re.findall('Entry requirements(.*)Visa requirements',rntry_requirements_en)[0]
#print(rntry_requirements_en)
except:
rntry_requirements_en = ''
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
ielts_desc = response.xpath('//*[@id="tab-Entry_Requirements"]/div/div[1]/div[1]/table[2]/tbody[2]/tr[1]/td[2]').extract()[0]
ielts_desc = remove_tags(ielts_desc)
#print(ielts_desc)
except:
ielts_desc = 'If English is not your first language or you have not previously studied in English, our usual requirement is the equivalent of an International English Language Testing System (IELTS Academic Test) score of 6.5, with not less than 6.0 in each of the sub-tests.'
try:
ielts = '6.5'
#i#elts = remove_tags(ielts)
#print(ielts)
except:
ielts = '6.5'
try:
ielts_l = re.findall('(\d.\d)',ielts_desc)[1]
#print(ielts_l)
#ielts_l = remove_tags(ielts_l)
except:
ielts_l = '6.0'
try:
ielts_s = ielts_l
except:
ielts_l = ''
try:
ielts_r = ielts_l
except:
ielts_l = ''
try:
ielts_w = ielts_l
except:
ielts_l = ''
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = ''
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = ''
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = ''
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = ''
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = ''
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = ''
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = ''
try:
work_experience_desc_en = response.xpath('//*[@id="content"]/div[13]/div/a/div/div[1]/p').extract()[0]
work_experience_desc_en = remove_tags(work_experience_desc_en)
#print(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
interview_desc_en = response.xpath('').extract()
interview_desc_en = remove_tags(interview_desc_en)
except:
interview_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
apply_desc_en = 'If English is not your first language or you have not previously studied in English, our usual requirement is the equivalent of an International English Language Testing System (IELTS Academic Test) score of 6.5, with not less than 6.0 in each of the sub-tests.'
apply_desc_en = remove_tags(apply_desc_en)
apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = '<p>birth certificate or passport IELTS English Language certificate degree certificate or transcript additional documentation requested by Registry. Please note: documentation that may be required by the Admissions Tutor in the relevant school and this should be sent directly to the school.</p>'
apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = response.xpath('//h1').extract()[0]
duration = remove_tags(duration)
duration = re.findall('(\d)',duration)[0]
#print(duration)
except:
duration = 'N/A'
#print(duration)
try:
other = response.xpath('//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a').extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath('//*[@id="tab-Entry_Requirements"]/div/div[1]/div[1]/table[1]/tbody/tr[11]/td[2]').extract()[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath('//*[@id="content"]/div[8]/div/div[2]').extract()[0]
alevel = remove_tags(alevel)
alevel = alevel.replace('\n','')
alevel = alevel.replace('\r\n','')
alevel = re.findall("UCAS.*?(\d\d\d)",alevel)[0]
#print(alevel)
except:
alevel = 'N/A'
#print(alevel)
try:
ucascode = response.xpath('//*[@id="content"]/div[2]/div[1]/dl/dd[5]').extract()[0]
ucascode = remove_tags(ucascode)
#print(ucascode)
except:
ucascode = ''
#print(ucascode)
try:
tuition_fee = response.xpath('//*[@id="content"]/div[8]/div/div[2]').extract()[0]
tuition_fee = remove_tags(tuition_fee)
tuition_fee = tuition_fee.replace(' ','')
tuition_fee = tuition_fee.replace('\n','')
tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
try:
assessment_en = response.xpath('//body').extract()[0]
assessment_en = remove_tags(assessment_en)
assessment_en = assessment_en.replace('\n','')
assessment_en = assessment_en.replace('\r\n','')
assessment_en = re.findall('Assessment(.*)Methods of assessment on this course',assessment_en)[0]
#assessment_en = assessment_en.replace(' ', ' ')
#assessment_en = assessment_en.replace('\n', '')
assessment_en = assessment_en.replace(' ','')
#assessment_en = assessment_en.replace(' ','')
assessment_en = "<div>"+assessment_en+'</div>'
#print(assessment_en)
except:
assessment_en = 'N/A'
#print(assessment_en)
try:
career_en = response.xpath('//body').extract()[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\n','')
career_en = career_en.replace('\r\n','')
career_en = re.findall('Careers and employability(.*)How to apply',career_en)[0]
#career_en = career_en.replace(' ', ' ')
#career_en = career_en.replace('\n', '')
career_en = career_en.replace(' ','')
career_en = career_en.replace(' ','')
career_en = "<div>"+career_en+'</div>'
#print(career_en)
except:
career_en = 'N/A'
#print(career_en)
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 1
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = 'all year'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_end
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = 0
item["toefl_l"] = 0
item["toefl_s"] = 0
item["toefl_r"] = 0
item["toefl_w"] = 0
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 5
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
item["alevel"] = alevel
item["ib"] = ib
item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["assessment_en"] = assessment_en
#item["apply_pre"] = ''
yield item
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'Abertay University'
try:
location = 'Dundee'
#location = remove_tags(location)
#print(location)
except:
location = 'N/A'
#print(location)
try:
department = response.xpath(
'//*[@id="course"]/section[1]/div/div/div/div[1]/div/div/div[3]/p[2]'
).extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
department = department.replace(' ', '')
department = department.replace('\n', '')
department = department.replace('Our Staff', '')
#print(department)
except:
department = ''
#print(department)
try:
degree_name = response.xpath(
'/html/body/div[1]/section/div/div/h1/span').extract()[0]
degree_name = remove_tags(degree_name)
degree_name = re.findall('(.* \(.*\))', degree_name)[0]
#degree_name = re.findall('(.*)\n.*',degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
degree_name = degree_name.replace(' ', '')
#print(degree_name)
except:
degree_name = 'BA'
#print(degree_name)
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath(
'/html/body/div[1]/section/div/div/h1').extract()[0]
programme_en = remove_tags(programme_en)
programme_en = programme_en.replace('\r\n', '')
programme_en = re.findall(
'(.*) .*',
programme_en)[0]
programme_en = programme_en.replace(' ', '')
#print(programme_en)
except:
programme_en = 'N/A'
#print(programme_en)
try:
overview_en = response.xpath(
'/html/body/div[1]/div/div[1]/section[1]/div/div/div[2]'
).extract()[0]
overview_en = remove_tags(overview_en)
overview_en = overview_en.replace(' ', '')
#overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n', '')
overview_en = overview_en.replace('\r\n', '')
overview_en = overview_en.replace('\n', '')
overview_en = '<div>' + overview_en + '</div>'
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = 'N/A'
#print(overview_en)
try:
start_date = '9'
#print(start_date)
except:
start_date = ''
try:
modules_en = response.xpath(
'//*[@id="Year1"]/div/div|//*[@id="Year2"]/div|/html/body/div[1]/div/div[1]/section[8]/div'
).extract()[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n', '\n')
modules_en = modules_en.replace('\r\n', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace('\n', '')
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]'
).extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ', '')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath(
'//*[@id="entryrequirements"]').extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = "<div>" + rntry_requirements_en + "</div>"
rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\n', '')
rntry_requirements_en = rntry_requirements_en.replace(' ', '')
#rntry_requirements_en =rntry_requirements_en.replace(' ','')
#print(rntry_requirements_en)
except:
rntry_requirements_en = 'N/A'
#print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = ''
except:
require_chinese_en = ''
try:
ielts_desc = response.xpath(
'//*[@id="entry-requirements"]/div/section[2]/div/ul/li'
).extract()[0]
ielts_desc = remove_tags(ielts_desc)
#print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
#ielts = '6.5'
#ielts =remove_tags(ielts)
#ielts = re.findall('IELTS(.*)',ielts)[0]
ielts = '6.5'
#print(ielts)
except:
ielts = 0
#print(ielts)
try:
#ielts_l = '5.5'
ielts_l = '6.0'
#print(ielts_l)
#ielts_l = remove_tags(ielts_l)
except:
ielts_l = 0
try:
ielts_s = ielts_l
except:
ielts_s = ielts_l
try:
ielts_r = ielts_l
except:
ielts_r = ielts_l
try:
ielts_w = ielts_l
except:
ielts_w = ielts_l
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = 0
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = 0
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = 0
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = 0
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = 0
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = 0
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = 0
try:
interview_desc_en = response.xpath(
'//*[@id="entry-requirements-accordion-0"]/div[1]').extract(
)[0]
interview_desc_en = remove_tags(interview_desc_en)
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
interview_desc_en = "<div>" + interview_desc_en + "</div>"
#print(interview_desc_en)
except:
interview_desc_en = 'N/A'
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath('//*[@id="section-5"]').extract()[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\r\n', '')
career_en = career_en.replace(' ', '')
career_en = career_en.replace('\n', '')
career_en = "<div><span>" + career_en + "</span></div>"
#print(career_en)
except:
career_en = ''
#print(career_en)
try:
apply_desc_en = 'https://www.abertay.ac.uk/courses/postgraduate-taught/request-prospectus/'
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = ''
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 0
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = response.xpath(
'/html/body/div[1]/div/div[1]/section[1]/div/div/div[1]/div/div[2]/span'
).extract()[0]
#duration = remove_tags(duration)
duration = remove_tags(duration)
#duration = re.findall('(\d) Years',duration)[0]
if '3' in duration:
duration = '3'
elif '2' in duration:
duration = '2'
elif '1' in duration:
duration = '1'
elif '4' in duration:
duration = '4'
elif '5' in duration:
duration = '5'
elif '1' in duration:
duration = '1'
elif 'two' in duration:
duration = '2'
else:
duration = '3'
#print(duration)
except:
duration = 3
#print(duration)
try:
other = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a'
).extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath(
'//*[@id="EntryToYear1"]/div/table/tbody/tr[5]/td[2]').extract(
)[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath(
'//*[@id="EntryToYear1"]/div/table/tbody/tr[3]/td[2]').extract(
)[0]
alevel = remove_tags(alevel)
#alevel = re.findall("(\w\w\w) at A Level",alevel)[0]
#print(alevel)
except:
alevel = 'N/A'
#print(alevel)
try:
ucascode = response.xpath(
'/html/body/div[1]/div/div[1]/section[1]/div/div/div[1]/div/div[4]/span|//*[@id="EntryToYear1"]/div/table/tbody/tr[3]/td[2]'
).extract()[0]
ucascode = remove_tags(ucascode)
#print(ucascode)
except:
ucascode = 'N/A'
#print(ucascode)
try:
tuition_fee = response.xpath(
'//div/div/div[3]/div/div[2]').extract()[0]
tuition_fee = remove_tags(tuition_fee)
tuition_fee = tuition_fee.replace('£', '')
tuition_fee = tuition_fee.replace(',', '')
tuition_fee = tuition_fee.replace('*', '')
tuition_fee = tuition_fee.replace(' ', '')
tuition_fee = tuition_fee.replace('\r\n', '')
tuition_fee = tuition_fee.replace('\n', '')
tuition_fee = re.findall('(\d\d\d\d\d)', tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#print(tuition_fee)
try:
assessment_en = response.xpath(
'/html/body/div[1]/div/div[1]/section[9]/div').extract()[0]
assessment_en = remove_tags(assessment_en)
assessment_en = assessment_en.replace('\r\n', '')
assessment_en = assessment_en.replace(' ', '')
assessment_en = assessment_en.replace('\n', '')
#assessment_en = re.findall('Learning and Assessment(.*)Accreditation',assessment_en)[0]
assessment_en = "<div><span>" + assessment_en + "</span></div>"
#print(assessment_en)
except:
assessment_en = 'N/A'
#print(assessment_en)
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 1
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 3
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
item["alevel"] = alevel
item["ib"] = ib
item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
item["assessment_en"] = assessment_en
#item["apply_pre"] = ''
yield item
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'Edinburgh Napier University'
try:
location = 'Merchiston'
#location = remove_tags(location)
#location = remove_tags(location)
#print(location)
except:
location = 'n/a'
#print(location)
try:
department = response.xpath('').extract()
department = remove_tags(department)
except:
department = ''
try:
degree_name = response.xpath(
'//*[@id="ctl22_centerdiv"]/div/h1/span[1]').extract()[0]
degree_name = remove_tags(degree_name)
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
#print(degree_name)
except:
degree_name = ''
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath(
'//*[@id="ctl22_centerdiv"]/div/h1/span[2]').extract()[0]
programme_en = remove_tags(programme_en)
#programme_en = re.findall("(.*)\(.*\)",programme_en)[0]
#programme_en = programme_en.replace('\n','')
#programme_en = programme_en.replace(' ','')
#print(programme_en)
except:
programme_en = ''
#print(programme_en)
try:
overview_en = response.xpath(
'//*[@id="tab-overview1"]/section/div[1]').extract()[0]
overview_en = remove_tags(overview_en)
overview_en = '<div>' + overview_en + '</div>'
overview_en = overview_en.replace(' ', '')
#overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n', '')
overview_en = overview_en.replace('\r\n', '')
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = ''
try:
start_date = '9'
#print(start_date)
except:
start_date = ''
try:
modules_en = response.xpath('//*[@id="pnlDetails"]').extract()[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n', '\n')
modules_en = modules_en.replace('\r\n', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace(' ', '')
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]'
).extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ', '')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath(
'//*[@id="tab0"]').extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = "<div>" + rntry_requirements_en + "</div>"
rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
rntry_requirements_en = rntry_requirements_en.replace(
' ', '')
#print(rntry_requirements_en)
except:
rntry_requirements_en = ''
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
ielts_desc = ''
#ielts_desc = remove_tags(ielts_desc)
#ielts_desc = ielts_desc.replace('\'','')
#ielts_desc = ielts_desc.replace('"','')
#ielts_desc = ielts_desc.replace('\xa0','')
# ielts_desc = ielts_desc.replace(' \n','')
# ielts_desc = ielts_desc.replace('\t\t','')
# ielts_desc = ielts_desc.replace('\\\n','')
#print(ielts_desc)
except:
ielts_desc = ''
try:
aa = response.xpath('//*[@id="tab4"]').extract()[0]
aa = remove_tags(aa)
except:
aa = 0
try:
ielts = re.findall('(\d.\d)', aa)[0]
#i#elts = remove_tags(ielts)
#print(ielts)
except:
ielts = 6.0
try:
ielts_l = re.findall('(\d.\d)', aa)[1]
#print(ielts_l)
#ielts_l = remove_tags(ielts_l)
except:
ielts_l = 5.5
try:
ielts_s = ielts_l
except:
ielts_s = 0
try:
ielts_r = ielts_l
except:
ielts_r = 0
try:
ielts_w = ielts_l
except:
ielts_w = 0
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = 0
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = 0
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = 0
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = 0
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = 0
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = 0
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = 0
try:
work_experience_desc_en = response.xpath(
'//*[@id="content"]/div[13]/div/a/div/div[1]/p').extract()[0]
work_experience_desc_en = remove_tags(work_experience_desc_en)
#print(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
interview_desc_en = response.xpath('').extract()
interview_desc_en = remove_tags(interview_desc_en)
except:
interview_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath(
'//*[@id="careers"]/section[2]/div[1]').extract()[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\r\n', '')
career_en = career_en.replace(' ', '')
career_en = "<div><span>" + career_en + "</span></div>"
#print(career_en)
except:
career_en = ''
try:
apply_desc_en = 'https://evision.napier.ac.uk/si/sits.urd/run/siw_ipp_lgn.login?'
apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = '<p>Your personal statement on your UCAS form is your chance to shine. It’s crucial to your application and can determine what offers you receive. When writing your statement: make sure it\'s styled in a formal manner and reads well write in a precise way using small paragraphs focus on your strengths, achievements and aspirations write about your skills and give examples explain why the course is interesting to you show that you\'ve done your research be truthful, accurate and enthusiastic check your spelling and grammar Be sure to include: your reasons for choosing the course relevant work or academic experience and skills personal interests that relate to the course your career goals any plans for taking a year out (if applicable)</p>'
apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 13
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = '1'
#duration = remove_tags(duration)
#duration = re.findall('(\d) Years',duration)[0]
#print(duration)
except:
duration = ''
#print(duration)
try:
other = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a'
).extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div[1]/table[1]/tbody/tr[11]/td[2]'
).extract()[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div/table[1]'
).extract()[0]
alevel = remove_tags(alevel)
alevel = re.findall("(\w\w\w) at A Level", alevel)[0]
#print(alevel)
except:
alevel = 'N/A'
#print(alevel)
try:
ucascode = response.xpath(
'//*[@id="courseInfo"]/div[2]/div[1]/p[2]').extract()[0]
ucascode = remove_tags(ucascode)
ucascode = ucascode.replace(' ', '')
ucascode = ucascode.replace('\r\n', '')
ucascode = ucascode.replace('\n', '')
#print(ucascode)
except:
ucascode = ''
#print(ucascode)
try:
tuition_fee = response.xpath(
'//*[@id="tab-fees"]/table/tbody/tr[3]/td[2]').extract()[0]
tuition_fee = remove_tags(tuition_fee)
tuition_fee = tuition_fee.replace('£', '')
tuition_fee = tuition_fee.replace(',', '')
tuition_fee = tuition_fee.replace('*', '')
tuition_fee = re.findall('(\d\d\d\d\d)', tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#print(tuition_fee)
try:
assessment_en = response.xpath(
'//*[@id="bodycontent_0_ctl04_assessments"]').extract()[0]
assessment_en = remove_tags(assessment_en)
assessment_en = assessment_en.replace(' ', ' ')
assessment_en = assessment_en.replace('\r\n', '')
assessment_en = assessment_en.replace('\n', '')
assessment_en = assessment_en.replace(' ',
'')
assessment_en = assessment_en.replace(
' ', '')
assessment_en = '<div>' + assessment_en + '</div>'
assessment_en = assessment_en.replace(' ', '')
print(assessment_en)
except:
assessment_en = 'N/A'
print(assessment_en)
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 1
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 4
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
item["alevel"] = alevel
item["ib"] = ib
item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
#item["apply_pre"] = ''
#item["assessment_en"]
item["assessment_en"] = assessment_en
yield item
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'University of Wales Trinity Saint David'
try:
location = response.xpath(
'/html/body/div[2]/div/div/div[2]/div[1]/div[2]/p[2]/a[2]/strong'
).extract()[0]
location = remove_tags(location)
#print(location)
except:
location = 'N/A'
#print(location)
try:
department = response.xpath(
'/html/body/div[2]/div/div/div[2]/div[1]/div[2]/p[2]/a[3]/strong'
).extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
department = department.replace(' ', '')
department = department.replace('\n', '')
department = department.replace('Our Staff', '')
#print(department)
except:
department = ''
#print(department)
try:
degree_name = response.xpath(
'/html/body/div[2]/section/div/div[2]/div/h1').extract()[0]
degree_name = remove_tags(degree_name)
degree_name = degree_name.split()[0]
#degree_name = re.findall('(.*)\n.*',degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
#degree_name = degree_name.replace(' ','')
#print(degree_name)
except:
degree_name = 'N/A'
#print(degree_name)
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath('//h1').extract()[0]
programme_en = remove_tags(programme_en)
#programme_en = re.findall(' (.*)',programme_en)[0]
programme_en = programme_en.replace(degree_name + ' ', '')
#programme_en = programme_en.replace(' ','')
#programme_en = programme_en.replace('\n', '')
#programme_en = re.findall((' '),'')[0]
#programme_en = re.findall("(.*)\(.*\)",programme_en)[0]
#programme_en = programme_en.replace('\n','')
#programme_en = programme_en.replace(' ','')
print(programme_en)
except:
programme_en = 'N/A'
print(programme_en)
try:
overview_en = response.xpath(
'//*[@id="collapseCourseOverview"]/div').extract()[0]
overview_en = remove_tags(overview_en)
overview_en = overview_en.replace(' ', '')
#overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n', '')
overview_en = overview_en.replace('\r\n', '')
overview_en = overview_en.replace('\n', '')
overview_en = '<div>' + overview_en + '</div>'
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = 'N/A'
#print(overview_en)
try:
start_date = '9'
#print(start_date)
except:
start_date = ''
try:
modules_en = response.xpath(
'//*[@id="collapseModules"]/div').extract()[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n', '\n')
modules_en = modules_en.replace('\r\n', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace('\n', '')
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]'
).extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ', '')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath(
'//*[@id="collapseEntryCriteria"]/div').extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = "<div>" + rntry_requirements_en + "</div>"
rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\n', '')
rntry_requirements_en = rntry_requirements_en.replace(' ', '')
#rntry_requirements_en =rntry_requirements_en.replace(' ','')
#print(rntry_requirements_en)
except:
rntry_requirements_en = 'N/A'
#print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = ''
except:
require_chinese_en = ''
try:
ielts_desc = 'An overall English Language level equivalent to an IELTS score of 6.5 (or equivalent) with no lower than 6.5 in reading and writing, and with no lower than 5.5 in listening or speaking.'
#print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
ielts = '6.5'
#ielts =remove_tags(ielts)
#ielts = re.findall('IELTS(.*)',ielts)[0]
#ielts = re.findall('(\d\.\d)',ielts)[0]
#print(ielts)
except:
ielts = 6.5
#print(ielts)
try:
ielts_l = 5.5
#print(ielts_l)
ielts_l = remove_tags(ielts_l)
except:
ielts_l = 5.5
try:
ielts_s = 5.5
except:
ielts_s = 5.5
try:
ielts_r = 6.5
except:
ielts_r = 6.5
try:
ielts_w = 6.5
except:
ielts_w = 6.5
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = 0
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = 0
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = 0
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = 0
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = 0
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = 0
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = 0
try:
interview_desc_en = response.xpath(
'//*[@id="entry-requirements-accordion-0"]/div[1]').extract(
)[0]
interview_desc_en = remove_tags(interview_desc_en)
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
interview_desc_en = "<div>" + interview_desc_en + "</div>"
#print(interview_desc_en)
except:
interview_desc_en = 'N/A'
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath(
'//*[@id="collapseCareerOpportunities"]/div').extract()[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\r\n', '')
career_en = career_en.replace(' ', '')
career_en = career_en.replace('\n', '')
career_en = "<div><span>" + career_en + "</span></div>"
#print(career_en)
except:
career_en = ''
#print(career_en)
try:
apply_desc_en = '<p>The Global Education Unit and academic staff frequently visit China throughout the year to meet prospective students and work with our academic partners. The university also work with International Student Recruitment Representatives and Recruitment Agents who can assist you in your application process. Alternatively, you can apply to the university directly, where an allocated international officer will help you through the applications process.</p>'
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = '<p>Undergraduate Programmes, applicants must have one of the following: Graduation Certificate from a Specialised College / School (Zhongzhuan) Chinese University / College Entrance Examination (Gaokao) Graduation Certificate (Zhuanke / Dazhuan / Gaozhi) – (may be considered for advanced entry to certain programmes) Successful completion of a recognised Foundation programme Postgraduate Programmes, applicants must have a Bachelor Degree. School certificates and certified transcripts must accompany your application.</p>'
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 0
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = response.xpath(
'/html/body/div[2]/div/div/div[2]/div[1]/div[2]/p[1]/strong[3]|/html/body/div[1]/div/div/div[2]/div[1]/div[2]/p[1]'
).extract()[0]
duration = remove_tags(duration)
#duration = remove_tags(duration)
#duration = re.findall('(\d) Years',duration)[0]
if '3 years' in duration:
duration = '3'
elif '4 years' in duration:
duration = '4'
elif '5 years' in duration:
duration = '5'
elif '6 years' in duration:
duration = '6'
elif '2 years' in duration:
duration = '2'
elif '1 year' in duration:
duration = '1'
elif 'two' in duration:
duration = '2'
else:
duration = '3'
#print(duration)
except:
duration = '0'
#print(duration)
try:
other = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a'
).extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div[1]/table[1]/tbody/tr[11]/td[2]'
).extract()[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div/table[1]'
).extract()[0]
alevel = remove_tags(alevel)
alevel = re.findall("(\w\w\w) at A Level", alevel)[0]
#print(alevel)
except:
alevel = 'CC'
#print(alevel)
try:
ucascode = response.xpath(
'/html/body/div[2]/div/div/div[2]/div[1]/div[2]/p[1]/strong[1]|/html/body/div[1]/div/div/div[2]/div[1]/div[2]/p[1]/strong[1]'
).extract()[0]
ucascode = remove_tags(ucascode)
#print(ucascode)
except:
ucascode = 'N/A'
#print(ucascode)
try:
tuition_fee = response.xpath(
'/html/body/div[2]/div/div/div[2]/div[1]/div[2]/p[2]').extract(
)[0]
tuition_fee = remove_tags(tuition_fee)
tuition_fee = tuition_fee.replace('£', '')
tuition_fee = tuition_fee.replace(',', '')
tuition_fee = tuition_fee.replace('*', '')
tuition_fee = tuition_fee.replace(' ', '')
tuition_fee = tuition_fee.replace('\r\n', '')
tuition_fee = tuition_fee.replace('\n', '')
tuition_fee = re.findall('(\d\d\d\d\d)', tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#print(tuition_fee)
try:
assessment_en = response.xpath(
'//*[@id="collapseAssessment"]/div').extract()[0]
assessment_en = remove_tags(assessment_en)
assessment_en = assessment_en.replace('\n', '')
assessment_en = assessment_en.replace('\r\n', '')
assessment_en = assessment_en.replace(' ', ' ')
assessment_en = "<p>" + assessment_en + "</p>"
#print(assessment_en)
except:
assessment_en = 'N/A'
#print(assessment_en)
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 1
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 9
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
item["alevel"] = alevel
item["ib"] = ib
item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
item["assessment_en"] = assessment_en
#item["apply_pre"] = ''
yield item
def parse(self, response):
pass
#print()
# print(response.url)
item = UcasItem()
university = 'susaikesi123'
try:
location = response.xpath(
'//*[@id="main_content"]/div[2]/div[1]/div/div[1]/div[1]/span'
).extract()[0]
location = remove_tags(location)
#print(location)
except:
location = 'N/A'
#print(location)
try:
department = response.xpath(
'//*[@id="main_content"]/div[2]/div[2]/div[3]/ul/li[4]/ul/li'
).extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
#department = department.replace(' ', '')
department = department.replace('\n', '')
#department = department.replace('Our Staff', '')
#print(department)
except:
department = ''
#print(department)
try:
degree_name = response.xpath(
'//*[@id="main_content"]/div[1]/div/div[2]/h1').extract()[0]
degree_name = remove_tags(degree_name)
degree_name = re.findall('.*- (.*)', degree_name)[0]
#degree_name = re.findall('\((.*)\).*',degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
#degree_name = degree_name.replace(' ','')
#print(degree_name)
except:
degree_name = 'N/A'
#print(degree_name)
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath(
'//*[@id="main"]/div[1]/section[1]/div/div[2]/div/div/h1'
).extract()[0]
programme_en = remove_tags(programme_en)
programme_en = programme_en.replace(degree_name, '')
#programme_en = programme_en.replace(' - University of Winchester ','')
#programme_en = programme_en.split()[1]
#programme_en = re.findall(' (.*)',programme_en)[0]
#programme_en = programme_en.replace(degree_name,'')
#programme_en = programme_en.replace(' ','')
#programme_en = programme_en.replace('\n', '')
#programme_en = re.findall((' '),'')[0]
#programme_en = re.findall("\(.*\)(.*)",programme_en)[0]
programme_en = programme_en.replace('\n', '')
programme_en = programme_en.replace(' ', '')
programme_en = programme_en.replace(' -', '')
#print(programme_en)
except:
programme_en = 'N/A'
#print(programme_en)
try:
overview_en = response.xpath(
'//*[@id="course-tab-1"]/section/p').extract()[0]
overview_en = remove_tags(overview_en)
#overview_en = re.findall('COURSE OVERVIEW(.*)',overview_en)[0]
overview_en = overview_en.replace(' ', '')
overview_en = overview_en.replace('\n\n', '\n')
overview_en = overview_en.replace('\n\n', '')
overview_en = overview_en.replace('\r\n', '')
#overview_en = overview_en.replace('\n','')
#overview_en = re.findall('COURSE OVERVIEW(.*)Careers',overview_en)[0]
overview_en = '<div>' + overview_en + '</div>'
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = 'N/A'
#print(overview_en)
try:
start_date = '2019-9'
#print(start_date)
except:
start_date = '11'
#print(start_date)
try:
#modules_en = response.xpath('//div[4]/div/div/div[1]/div[5]/div/div[2]/p').extract()[0]
modules_en = response.xpath('//*[@id="structure"]').extract()[0]
#modules_en = re.findall('str.replace(/<([a-zA-Z]+)\s*[^><]*>/g,"<$1>")',aa)[0]
#modules_en = aa.replace(modules_en,'')
modules_en = remove_tags(modules_en)
#modules_en = remove_tags(modules_en,keep=('div','p','span','ul','p','li'))
#modules_en = modules_en.replace(' class="tabs-panel is-active" id="tab-content-year-1"','')
modules_en = modules_en.replace(' ', ' ')
#modules_en = modules_en.replace('\n\n','')
#modules_en = modules_en.replace('\n','')
modules_en = modules_en.replace('\r\n', ' ')
modules_en = '<div><p>' + modules_en + "</p></div>"
# overview_en = re.findall('COURSE OVERVIEW(.*)',overview_en)[0]
# modules_en = modules_en.replace(' ', ' ')
# modules_en = modules_en.replace('\n\n', '\n')
# modules_en = modules_en.replace('\n\n', '')
# modules_en = modules_en.replace('\r\n', '')
# modules_en = modules_en.replace('\n', '')
#modules_en = re.findall('Year 1(.*)in Year 1', modules_en)[0]
#modules_en = '<div>' + modules_en + '</div>'
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]'
).extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ', '')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath(
'//*[@id="entry-requirements"]').extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = re.findall('\d\d',
rntry_requirements_en)[0]
rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\n', '')
rntry_requirements_en = rntry_requirements_en.replace(' ', '')
#rntry_requirements_en = re.findall('ENTRY REQUIREMENTS(.*)Visit us',rntry_requirements_en)[0]
#rntry_requirements_en = "<div>"+rntry_requirements_en+"</div>"
#rntry_requirements_en =rntry_requirements_en.replace(' ','')
if '79' in rntry_requirements_en:
rntry_requirements_en = '79'
else:
rntry_requirements_en = 'N/A'
#print(rntry_requirements_en)
except:
rntry_requirements_en = 'N/A'
#print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = ''
except:
require_chinese_en = ''
try:
ielts_desc = response.xpath('//tr[1]/td[2]').extract()[0]
ielts_desc = remove_tags(ielts_desc)
#print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
#ielts = '6.5'
#ielts =remove_tags(ielts)
if 'MSc' in degree_name:
ielts = '7.0'
ielts_s = '7.0'
ielts_l = '6.0'
ielts_r = '6.0'
ielts_w = '6.0'
toefl = '95'
toefl_s = ''
toefl_l = ''
toefl_r = 22
toefl_w = 21
else:
toefl = ''
#ielts =
#print(ielts)
except:
ielts = 0
#print(ielts)
try:
interview_desc_en = response.xpath(
'//*[@id="entry-requirements-accordion-0"]/div[1]').extract(
)[0]
interview_desc_en = remove_tags(interview_desc_en)
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
interview_desc_en = "<div>" + interview_desc_en + "</div>"
#print(interview_desc_en)
except:
interview_desc_en = 'N/A'
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath(
'//*[@id="tab-Employability"]/div/div[2]/div[2]').extract()[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\r\n', '')
career_en = career_en.replace(' ', '')
career_en = career_en.replace('\n', '')
career_en = "<div><span>" + career_en + "</span></div>"
print(career_en)
except:
career_en = ''
print(career_en)
try:
apply_desc_en = '<p>Undergraduate applicants Applications for full-time undergraduate courses must be made through UCAS Apply. International students can also apply through UCAS with the assistance of one of Kent\'s overseas representatives. See how to apply for undergraduate study. If you are applying for a subject-specific foundation programme, you apply in the same way as for a three-year undergraduate degree. Applications for our International Foundation Programme (IFP) can be made: directly to Kent via the International Foundation Programme site via one of Kent\'s overseas representatives via UCAS. If you are submitting other applications to Kent through UCAS, your IFP application must also be made through UCAS. Important dates Students from the EU are advised to apply by 15 January Students from outside the EU should aim for 15 January to guarantee consideration. However, if places are left, we may consider applications up to 30 June. Late applications A process called \'Clearing\' takes place in July and August, running from 1st July annually. We will provide an updated list all of the Kent degrees that still have places available once Clearing opens – this can give you another chance to apply. For more information on how the Clearing process works and how to apply, please see Clearing at Kent for details. Postgraduate applicants See how to apply for postgraduate study.</p>'
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = '<p>For entry to a Kent postgraduate degree programme (Master’s), Chinese students typically need to have completed a Bachelor Degree (Xueshi) at a recognised institution. Exact requirements will depend on the postgraduate degree you are applying for and the undergraduate degree you have studied. For programmes that require a 2:1 we usually ask for a Bachelor degree (Xueshi) from a 211 university with a final grade of 70%. For Bachelor degrees from other recognised institutions you will need to achieve a final grade of 75% For programmes that require a 2:2 we usually ask for a Bachelor degree (Xueshi) from a 211 university with a final grade of 65%. For Bachelor degrees from other recognised institutions you will need to achieve a final grade of 70% Applicants with relevant work experience may be considered with lower grades. Some, but not all, postgraduate programmes require your undergraduate degree to have a related major. Some postgraduate programmes may require work experience in a relevant field or at a certain level.</p>'
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 0
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = response.xpath(
'//*[@id="main_content"]/div[2]/div[2]/div[3]/ul/li[3]/ul/li'
).extract()[0]
duration = remove_tags(duration)
#duration = remove_tags(duration)
#duration = re.findall('(\d) Years',duration)[0]
if 'One' in duration:
duration = '1'
elif '1' in duration:
duration = '1'
elif '12' in duration:
duration = '1'
elif 'Two' in duration:
duration = '2'
elif '2' in duration:
duration = '2'
elif '1' in duration:
duration = '1'
elif 'two' in duration:
duration = '2'
else:
duration = '1'
#print(duration)
except:
duration = 1
#print(duration)
try:
other = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a'
).extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath(
'//*[@id="tab-Entry_Requirements"]').extract()[0]
ib = remove_tags(ib)
ib = re.findall('(\d\d) points overall. ', ib)[0]
ib = ib + ' points overall. '
#print(ib)
except:
ib = 'N/A'
# print(ib)
try:
alevel = response.xpath(
'//*[@id="primary-content"]/div[3]/div[1]/div[1]/div[2]/p[1]/span'
).extract()[0]
alevel = remove_tags(alevel)
#alevel = re.findall("(\w\w\w) at A Level",alevel)[0]
#print(alevel)
except:
alevel = 'N/A'
#print(alevel)
try:
ucascode = response.xpath(
'//*[@id="primary-content"]/div[3]/div[1]/div[2]/ul[1]/li[1]/span'
).extract()[0]
ucascode = remove_tags(ucascode)
#print(ucascode)
except:
ucascode = 'N/A'
#print(ucascode)
try:
tuition_fee = response.xpath(
'//*[@id="fees"]/table/tbody/tr/td[3]').extract()[0]
# tuition_fee = remove_tags(tuition_fee)
# tuition_fee = tuition_fee.replace('£','')
# tuition_fee = tuition_fee.replace(',','')
# tuition_fee = tuition_fee.replace('*','')
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\r\n','')
# tuition_fee = tuition_fee.replace('\n','')
#
tuition_fee = re.findall('(\d\d\d\d\d)', tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#(tuition_fee)
try:
assessment_en = response.xpath(
'//*[@id="detail"]/div/div/div[3]').extract()[0]
assessment_en = remove_tags(assessment_en)
assessment_en = assessment_en.replace('\r\n', '')
assessment_en = assessment_en.replace(' ', '')
assessment_en = assessment_en.replace('\n', '')
assessment_en = assessment_en.replace(' ', '')
assessment_en = assessment_en.replace(' ', '')
#assessment_en = re.findall('Assessment:(.*)Entry Requirements:',assessment_en)[0]
assessment_en = "<div>" + assessment_en + "</div>"
#print(assessment_en)
except:
assessment_en = 'N/A'
#print(assessment_en)
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 1
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = '7-31'
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = '6.5'
item["ielts_l"] = '5.5'
item["ielts_s"] = '5.5'
item["ielts_r"] = '6.0'
item["ielts_w"] = '6.0'
item["toefl_code"] = '0826'
item[
"toefl_desc"] = 'General postgraduate programmes 6.5 overall (with a minimum of 6.0 in R&W; 5.5 in S&L)90 overall (with a minimum of 22 in R; 21 in W; 17 in L; 20 in S)'
item["toefl_l"] = '17'
item["toefl"] = '90'
item["toefl_s"] = '20'
item["toefl_r"] = '22'
item["toefl_w"] = '21'
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 3
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
item["alevel"] = alevel
item["ib"] = ''
item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item[
"require_chinese_en"] = '<p>For entry to a Kent postgraduate degree programme (Master’s), Chinese students typically need to have completed a Bachelor Degree (Xueshi) at a recognised institution. Exact requirements will depend on the postgraduate degree you are applying for and the undergraduate degree you have studied. For programmes that require a 2:1 we usually ask for a Bachelor degree (Xueshi) from a 211 university with a final grade of 70%. For Bachelor degrees from other recognised institutions you will need to achieve a final grade of 75% For programmes that require a 2:2 we usually ask for a Bachelor degree (Xueshi) from a 211 university with a final grade of 65%. For Bachelor degrees from other recognised institutions you will need to achieve a final grade of 70% Applicants with relevant work experience may be considered with lower grades. Some, but not all, postgraduate programmes require your undergraduate degree to have a related major. Some postgraduate programmes may require work experience in a relevant field or at a certain level.</p>'
item["assessment_en"] = assessment_en
item["teach_time"] = ''
item["teach_type"] = ''
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'Canterbury Christ Church University'
try:
location = 'Canterbury'
location = remove_tags(location)
#print(location)
except:
location = 'N/A'
#print(location)
try:
department = response.xpath('//ul[@class="course-department"]').extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
department = department.replace(' ', '')
department = department.replace('\n', '')
department = department.replace('Our Staff', '')
#print(department)
except:
department = 'N/A'
#print(department)
try:
degree_name = response.xpath('//*[@id="form1"]/div[3]/div/div/div[1]/div[1]/h1/span[1]').extract()[0]
degree_name = remove_tags(degree_name)
#degree_name = degree_name.split()[0]
#degree_name = re.findall('(.*)\n.*',degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
degree_name = degree_name.replace(' ','')
#print(degree_name)
except:
degree_name = 'N/A'
#print(degree_name)
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath('///*[@id="form1"]/div[3]/div/div/div[1]/div[1]/h1/span[2]').extract()[0]
programme_en = remove_tags(programme_en)
programme_en = re.findall(' (.*)',programme_en)[0]
#programme_en = programme_en.replace(degree_name,'')
#programme_en = programme_en.replace(' ','')
#programme_en = programme_en.replace('\n', '')
#programme_en = re.findall((' '),'')[0]
#programme_en = re.findall("(.*)\(.*\)",programme_en)[0]
#programme_en = programme_en.replace('\n','')
#programme_en = programme_en.replace(' ','')
#print(programme_en)
except:
programme_en = ''
#print(programme_en)
try:
overview_en = response.xpath('//*[@id="collapseOne"]/div').extract()[0]
overview_en = remove_tags(overview_en)
overview_en = overview_en.replace(' ','')
#overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n','')
overview_en = overview_en.replace('\r\n','')
overview_en = overview_en.replace('\n','')
overview_en = '<div>' + overview_en + '</div>'
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = 'N/A'
#print(overview_en)
try:
start_date = '9'
#print(start_date)
except:
start_date = ''
try:
modules_en = response.xpath('//*[@id="collapseThree"]/div').extract()[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n','\n')
modules_en = modules_en.replace('\r\n','')
modules_en = modules_en.replace(' ','')
modules_en = modules_en.replace(' ','')
modules_en = modules_en.replace('\n','')
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath('//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]').extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ','')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath('//*[@id="form1"]/div[3]/div/div/div[2]/div/ul[3]').extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = "<div>"+rntry_requirements_en+"</div>"
rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\n', '')
rntry_requirements_en = rntry_requirements_en.replace(' ','')
#rntry_requirements_en =rntry_requirements_en.replace(' ','')
#print(rntry_requirements_en)
except:
rntry_requirements_en = 'N/A'
#print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = ' <p>Bachelor degree (Xueshi) For programmes requiring a 2:1 : - A minimum final grade of 75% or equivalent - A minimum final grade of 70% or equivalent from one of the 211 universities For programmes requiring a 2:2: - A minimum final grade of 70% or equivalent - A minimum final grade of 65% or equivalent from one of the 211 universities Read more at https://www.canterbury.ac.uk//study-here/international/your-country/china.aspx#6IcURYBGrdAXt2Fs.99H</p>'
except:
require_chinese_en = ' <p>Bachelor degree (Xueshi) For programmes requiring a 2:1 : - A minimum final grade of 75% or equivalent - A minimum final grade of 70% or equivalent from one of the 211 universities For programmes requiring a 2:2: - A minimum final grade of 70% or equivalent - A minimum final grade of 65% or equivalent from one of the 211 universities Read more at https://www.canterbury.ac.uk//study-here/international/your-country/china.aspx#6IcURYBGrdAXt2Fs.99H</p>'
try:
ielts_desc = '<p>.For postgraduate entry, you will need a minimum of IELTS 6.5 or equivalent, with no less than 6.0 in Writing and no less than 5.5 in Reading, Speaking and Listening. We are able to accept other English qualifications, including Bachelor\'s and Master\'s degrees depending upon the country of issue.</p>'
#print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
ielts = 6.5
#print(ielts)
except:
ielts = 6.5
#print(ielts)
try:
ielts_l = 5.5
#print(ielts_l)
#ielts_l = remove_tags(ielts_l)
except:
ielts_l = 5.5
try:
ielts_s = ielts_l
except:
ielts_s = 0
try:
ielts_r = ielts_l
except:
ielts_r = 0
try:
ielts_w = 6.0
except:
ielts_w = 0
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = 0
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = 0
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = 0
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = 0
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = 0
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = 0
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = 0
try:
interview_desc_en = response.xpath('//*[@id="entry-requirements-accordion-0"]/div[1]').extract()[0]
interview_desc_en = remove_tags(interview_desc_en)
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
interview_desc_en = "<div>" + interview_desc_en + "</div>"
#print(interview_desc_en)
except:
interview_desc_en = 'N/A'
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath('//*[@id="your-future-career-5"]/div[1]/div').extract()[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\r\n','')
career_en = career_en.replace(' ','')
career_en = career_en.replace('\n','')
career_en = "<div><span>" + career_en + "</span></div>"
#print(career_en)
except:
career_en = ''
#print(career_en)
try:
apply_desc_en = '<p>Choose your subject area* Download and complete application form Submit your application form with evidence of your qualifications Receive conditional offer letter Meet any offer conditions Apply for accommodation Receive final confirmation of offer Accept confirmation Arrive with original documents confirming your qualifications Register on course Begin course</p>'
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = '<div>Make sure you send a copy of your first degree (or equivalent) certificate and transcript. Don\'t forget you will need evidence of your level of English language (IELTS, TOEFL, etc). You will also need to send at least one academic reference. Some degree programmes have additional requirements - check the relevant academic department web pages for details. Never send original documents - send copies only. There are special arrangements for international students applying for courses which require an interview or audition. These will be detailed in your conditional offer letter.</div>'
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 0
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = response.xpath('//*[@id="form1"]/div[3]/div/div/div[2]/div/ul[3]/li').extract()[0]
duration = remove_tags(duration)
duration = re.findall('(\d) years',duration)[0]
#print(duration)
except:
duration = ''
#print(duration)
try:
other = response.xpath('//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a').extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath('//*[@id="tab-Entry_Requirements"]/div/div[1]/div[1]/table[1]/tbody/tr[11]/td[2]').extract()[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath('//*[@id="form1"]/div[3]/div/div/div[2]/div[1]/ul[5]/li').extract()[0]
alevel = remove_tags(alevel)
#alevel = re.findall("(\w\w\w) at A Level",alevel)[0]
#print(alevel)
except:
alevel = 'N/A'
#print(alevel)
try:
ucascode = response.xpath('//*[@id="form1"]/div[3]/div/div/div[2]/div/ul[1]/li[1]/span|//*[@id="form1"]/div[3]/div/div/div[2]/div/ul[1]/li[2]/span').extract()[0]
ucascode = remove_tags(ucascode)
#print(ucascode)
except:
ucascode = 'N/A'
#print(ucascode)
try:
tuition_fee = response.xpath('//tbody/tr[2]/td[2]').extract()[0]
tuition_fee = remove_tags(tuition_fee)
tuition_fee = tuition_fee.replace('£','')
tuition_fee = tuition_fee.replace(',','')
tuition_fee = tuition_fee.replace('*','')
tuition_fee = tuition_fee.replace(' ','')
tuition_fee = tuition_fee.replace('\r\n','')
tuition_fee = tuition_fee.replace('\n','')
tuition_fee = re.findall('(\d\d\d\d\d)',tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#print(tuition_fee)
try:
assessment_en = response.xpath('//*[@id="collapseEight"]/div/p|//*[@id="collapseSeven"]/div').extract()[0]
assessment_en = remove_tags(assessment_en)
assessment_en = assessment_en.replace('\n','')
assessment_en = "<div>" + assessment_en + "</div>"
#print(assessment_en)
except:
assessment_en = 'N/A'
#print(assessment_en)
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 2
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 4
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
item["alevel"] = alevel
item["ib"] = ib
item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
item["assessment_en"] = assessment_en
#item["apply_pre"] = ''
yield item
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'Arts University Bournemouth'
try:
location = 'Woodlane, Falmouth'
location = remove_tags(location)
#print(location)
except:
location = 'N/A'
#print(location)
try:
department = response.xpath(
'/html/body/div[1]/div/div/div[2]/div[1]/div[2]/p[2]/a[3]/strong'
).extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
department = department.replace(' ', '')
department = department.replace('\n', '')
department = department.replace('Our Staff', '')
#print(department)
except:
department = ''
#print(department)
try:
degree_name = response.xpath(
'//*[@id="start-of-content"]/div[1]/div[1]/div[1]/h1').extract(
)[0]
degree_name = remove_tags(degree_name)
degree_name = degree_name.split()[-1]
#degree_name = re.findall('(.*)\n.*',degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
degree_name = degree_name.replace(' ', '')
#print(degree_name)
except:
degree_name = 'N/A'
#print(degree_name)
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath(
'//*[@id="content"]/main/div/div[1]/article/header/h1'
).extract()[0]
programme_en = remove_tags(programme_en)
#programme_en = re.findall(' (.*)',programme_en)[0]
#programme_en = programme_en.replace(degree_name,'')
programme_en = programme_en.replace(' ', '')
#programme_en = programme_en.replace('\n', '')
#programme_en = re.findall((' '),'')[0]
#programme_en = re.findall("(.*)\(.*\)",programme_en)[0]
#programme_en = programme_en.replace('\n','')
#programme_en = programme_en.replace(' ','')
#print(programme_en)
except:
programme_en = 'N/A'
#print(programme_en)
try:
overview_en = response.xpath('//*[@id="overview"]').extract()[0]
overview_en = remove_tags(overview_en)
overview_en = overview_en.replace(' ', '')
#overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n', '')
overview_en = overview_en.replace('\r\n', '')
overview_en = overview_en.replace('\n', '')
overview_en = '<div>' + overview_en + '</div>'
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = 'N/A'
#print(overview_en)
try:
start_date = '9'
#print(start_date)
except:
start_date = ''
try:
modules_en = response.xpath(
'//*[@id="study"]/div[2]/div').extract()[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n', '\n')
modules_en = modules_en.replace('\r\n', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace('\n', '')
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]'
).extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ', '')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = '<div>APPLICANTS WILL USUALLY HAVE A 2:1 OR EQUIVALENT UNDERGRADUATE LEVEL QUALIFICATION AND MUST ALSO HOLD A RELEVANT ENGLISH LANGUAGE QUALIFICATION. For overseas applicants, we accept a range of English Language qualifications, including the following qualifications for MA and Master of Architecture (MArch) courses: Cambridge Certificate of Advanced English (CAE) grade C Cambridge Certificate of Proficiency in English (CPE) grade C Certificate of Attainment in English (London Board) level 5 or 6 IELTS score of 6.5 or above Pearson PTE score of 54 and over, with a score of 51 in each component. IELTS, Pearson and TOEFL scores must be less than two years old at the time the course commences to be valid. We may also accept TOEFL iBT scores of 80 or above for entry to degree courses; however we would prefer one of the above qualifications. TOEFL is a test of American English and is not classified as a Secure English Language Test (SELT) for Tier 4 Visa applicants.</div>'
#rntry_requirements_en = remove_tags(rntry_requirements_en)
# rntry_requirements_en = "<div>"+rntry_requirements_en+"</div>"
# rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
# rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
# rntry_requirements_en = rntry_requirements_en.replace('\n', '')
# rntry_requirements_en = rntry_requirements_en.replace(' ','')
# #rntry_requirements_en =rntry_requirements_en.replace(' ','')
#print(rntry_requirements_en)
except:
rntry_requirements_en = 'N/A'
#print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = ''
except:
require_chinese_en = ''
try:
ielts_desc = 'IELTS score of 6.5 or above'
#print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
ielts = '6.5'
#ielts =remove_tags(ielts)
#ielts = re.findall('IELTS(.*)',ielts)[0]
#ielts = re.findall('(\d\.\d)',ielts)[0]
#print(ielts)
except:
ielts = 6.5
#print(ielts)
try:
ielts_l = 6.0
#print(ielts_l)
ielts_l = remove_tags(ielts_l)
except:
ielts_l = 6.0
try:
ielts_s = 6.0
except:
ielts_s = 6.0
try:
ielts_r = 6.0
except:
ielts_r = 6.0
try:
ielts_w = 6.0
except:
ielts_w = 6.0
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = 0
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = 0
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = 0
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = 0
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = 0
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = 0
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = 0
try:
interview_desc_en = response.xpath(
'//*[@id="entry-requirements-accordion-0"]/div[1]').extract(
)[0]
interview_desc_en = remove_tags(interview_desc_en)
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
interview_desc_en = "<div>" + interview_desc_en + "</div>"
#print(interview_desc_en)
except:
interview_desc_en = 'N/A'
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath(
'//*[@id="collapseCareerOpportunities"]/div').extract()[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\r\n', '')
career_en = career_en.replace(' ', '')
career_en = career_en.replace('\n', '')
career_en = "<div><span>" + career_en + "</span></div>"
#print(career_en)
except:
career_en = ''
#print(career_en)
try:
apply_desc_en = '<p>You can apply from late October and there is no set deadline for applications. It’s worth remembering that overseas applicants who pay one term’s fees in advance by 1st August will receive a 10% discount. We encourage you to apply early, so that you have time to organise accommodation and to arrange for a visa, if required, before you enrol.</p>'
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = ''
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 0
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = '1'
except:
duration = '0'
#print(duration)
try:
other = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a'
).extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div[1]/table[1]/tbody/tr[11]/td[2]'
).extract()[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div/table[1]'
).extract()[0]
alevel = remove_tags(alevel)
alevel = re.findall("(\w\w\w) at A Level", alevel)[0]
#print(alevel)
except:
alevel = 'CC'
#print(alevel)
try:
ucascode = response.xpath(
'/html/body/div[3]/div[1]/div/div/div[2]/div/div[1]/div[1]/div[2]'
).extract()[0]
ucascode = remove_tags(ucascode)
#print(ucascode)
except:
ucascode = ''
#print(ucascode)
try:
tuition_fee = response.xpath(
'/html/body/div[4]/div/section/div[2]/div[2]/div/div[2]/div[1]/div[2]/span[2]/a'
).extract()[0]
tuition_fee = remove_tags(tuition_fee)
tuition_fee = tuition_fee.replace('£', '')
tuition_fee = tuition_fee.replace(',', '')
tuition_fee = tuition_fee.replace('*', '')
tuition_fee = tuition_fee.replace(' ', '')
tuition_fee = tuition_fee.replace('\r\n', '')
tuition_fee = tuition_fee.replace('\n', '')
tuition_fee = re.findall('(\d\d\d\d\d)', tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#print(tuition_fee)
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 2
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 1
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
#item["alevel"] = alevel
#item["ib"] = ib
#item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
#item["apply_pre"] = ''
yield item
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'University of Worcester'
try:
location = 'Worcester'
location = remove_tags(location)
#print(location)
except:
location = 'N/A'
#print(location)
try:
department = response.xpath(
'/html/body/div[1]/div/div/div[2]/div[1]/div[2]/p[2]/a[3]/strong'
).extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
department = department.replace(' ', '')
department = department.replace('\n', '')
department = department.replace('Our Staff', '')
#print(department)
except:
department = ''
#print(department)
try:
degree_name = response.xpath('//h1').extract()[0]
degree_name = remove_tags(degree_name)
degree_name = degree_name.replace('(Hons)', '')
degree_name = degree_name.split()[-1]
#degree_name = re.findall('(.*)\n.*',degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
#degree_name = degree_name.replace(' ','')
#print(degree_name)
except:
degree_name = 'N/A'
#print(degree_name)
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath('//h1').extract()[0]
programme_en = remove_tags(programme_en)
#programme_en = re.findall(' (.*)',programme_en)[0]
programme_en = programme_en.replace(degree_name, '')
programme_en = programme_en.replace(' ', ' ')
#programme_en = programme_en.replace('\n', '')
#programme_en = re.findall((' '),'')[0]
#programme_en = re.findall("(.*)\(.*\)",programme_en)[0]
#programme_en = programme_en.replace('\n','')
#programme_en = programme_en.replace(' ','')
#print(programme_en)
except:
programme_en = 'N/A'
#print(programme_en)
try:
overview_en = response.xpath(
'//*[@id="section-1"]/div[1]/div[1]/ul').extract()[0]
overview_en = remove_tags(overview_en)
overview_en = overview_en.replace(' ', '')
#overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n', '')
overview_en = overview_en.replace('\r\n', '')
overview_en = overview_en.replace('\n', '')
overview_en = '<div>' + overview_en + '</div>'
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = 'N/A'
#print(overview_en)
try:
start_date = '9'
#print(start_date)
except:
start_date = ''
try:
modules_en = response.xpath('//*[@id="section-3"]').extract()[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n', '\n')
modules_en = modules_en.replace('\r\n', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace('\n', '')
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]'
).extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ', '')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath(
'//*[@id="section-2"]/div/div[2]').extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = "<div>" + rntry_requirements_en + "</div>"
rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\n', '')
rntry_requirements_en = rntry_requirements_en.replace(' ', '')
#rntry_requirements_en =rntry_requirements_en.replace(' ','')
#print(rntry_requirements_en)
except:
rntry_requirements_en = 'N/A'
#print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = ''
except:
require_chinese_en = ''
try:
ielts_desc = 'Postgraduate courses usually require a minimum IELTS of 6.5 (with no less than 5.5 in any component)'
ielts_desc = remove_tags(ielts_desc)
#print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
ielts = '6.5'
#ielts =remove_tags(ielts)
#ielts = re.findall('IELTS(.*)',ielts)[0]
#ielts = re.findall('(\d\.\d)',ielts)[0]
#print(ielts)
except:
ielts = 0
#print(ielts)
try:
ielts_l = '5.5'
ielts = re.findall('(\d\.\d)', ielts)[1]
#print(ielts_l)
#ielts_l = remove_tags(ielts_l)
except:
ielts_l = 0
try:
ielts_s = ielts_l
except:
ielts_s = ielts_l
try:
ielts_r = ielts_l
except:
ielts_r = ielts_l
try:
ielts_w = ielts_l
except:
ielts_w = ielts_l
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = 0
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = 0
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = 0
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = 0
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = 0
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = 0
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = 0
try:
interview_desc_en = response.xpath(
'//*[@id="entry-requirements-accordion-0"]/div[1]').extract(
)[0]
interview_desc_en = remove_tags(interview_desc_en)
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
interview_desc_en = "<div>" + interview_desc_en + "</div>"
#print(interview_desc_en)
except:
interview_desc_en = 'N/A'
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath('//*[@id="section-5"]').extract()[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\r\n', '')
career_en = career_en.replace(' ', '')
career_en = career_en.replace('\n', '')
career_en = "<div><span>" + career_en + "</span></div>"
#print(career_en)
except:
career_en = ''
#print(career_en)
try:
apply_desc_en = '<div>Undergraduate degrees (BSc, BA, HND and Foundation degrees) If you are applying for a full-time undergraduate course, EU/ Non-EU students are strongly advised to apply online through the Universities & Colleges Admissions Service (UCAS). If you are using The Common Application, you can add the University of Worcester to your list of colleges via this link and complete the application there. You can get in touch with us before you apply to get advice on the offer you are likely to receive. For further details on when to apply, please see the Undergraduate How to apply section. Postgraduate degrees (MA, MSc, MBA) If you are applying for a place on a postgraduate course please apply directly to Worcester using the Overseas Application for Admission Form. For further details on when to apply, please refer to the Taught Postgraduate How to apply section. Applying through University of Worcester Overseas Representatives You also have the option to use the services of our recognised representatives overseas. They can give you advice and guide you through the process of applying. To see who we work with in your country, please visit the Overseas Representatives page. What do you need to include in your application? In order for us to process your application successfully, you will need to provide the following documents: All academic transcripts Copies of degree/diploma certificates Up to two academic references (depending on your course level and subject) An English language test score (IELTS/Cambridge Advanced, Pearson) Your personal statement Copy of your current passport or ID card It is important that you send this information to us - either paper copies by mail or scanned and emailed - as soon as possible. The International Team and Admissions Office will process your application and, if the application is complete, will endeavour to respond with a decision as soon as we can. Receiving an offer All applications to the University of Worcester are carefully considered by an admissions tutor (an academic member of staff from the relevant course area), whose job is to view applications. They will make one of the following offers or responses to you through UCAS: an Unconditional offer, (you have achieved the entry requirements) a Conditional offer (you need to achieve specified entry requirements before you are accepted) regretfully reject your application (this does not prevent you from applying again in a subsequent year) For full details about the different stages of the application process, please refer to the What happens to my application pages. Good luck with your preparation and we hope that you will become part of our International Community at the University of Worcester.</div>'
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = '<p>All academic transcripts Copies of degree/diploma certificates Up to two academic references (depending on your course level and subject) An English language test score (IELTS/Cambridge Advanced, Pearson) Your personal statement Copy of your current passport or ID card</p>'
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 0
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
# duration = response.xpath('/html/body/div/div/div/div[2]/div[1]/div[2]/p[1]/strong').extract()[0]
# duration = remove_tags(duration)
duration = 3
#duration = re.findall('(\d) Years',duration)[0]
# if '36' in duration:
# duration = '3'
# elif '16' in duration:
# duration = '1'
# elif '12' in duration:
# duration = '1'
# elif '3' in duration:
# duration = '3'
# elif '2' in duration:
# duration = '2'
# elif '1' in duration:
# duration = '1'
# elif 'two' in duration:
# duration = '2'
# else:
# duration = '1'
# #print(duration)
except:
duration = 0
#print(duration)
try:
other = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a'
).extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div[1]/table[1]/tbody/tr[11]/td[2]'
).extract()[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div/table[1]'
).extract()[0]
alevel = remove_tags(alevel)
alevel = re.findall("(\w\w\w) at A Level", alevel)[0]
#print(alevel)
except:
alevel = 'N/A'
#print(alevel)
try:
ucascode = response.xpath('//body').extract()[0]
ucascode = remove_tags(ucascode)
ucascode = ucascode.replace('\r\n', '')
ucascode = ucascode.replace('\n', '')
ucascode = ucascode.replace(' ', ' ')
if 'Apply through UCAS' in ucascode:
ucascode = re.findall(
'Apply through UCAS.*- (\w\w\w\w)',
ucascode)[0] or re.findall(
'\(.*\)(\w\w\w\w)', ucascode)[0] or re.findall(
'Apply through UCAS.*– (\w\w\w\w)', ucascode)[0]
elif 'Apply through UCAS' not in ucascode:
ucascode = response.xpath(
'//*[@id="section-7"]/div[2]/div[2]/p[1]').extract()[0]
ucascode = remove_tags(ucascode)
else:
ucascode = 'N/A'
print(ucascode)
except:
ucascode = 'N/A'
print(ucascode)
try:
tuition_fee = '12900'
# tuition_fee = remove_tags(tuition_fee)
# tuition_fee = tuition_fee.replace('£','')
# tuition_fee = tuition_fee.replace(',','')
# tuition_fee = tuition_fee.replace('*','')
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\r\n','')
# tuition_fee = tuition_fee.replace('\n','')
#
# tuition_fee = re.findall('(\d\d\d\d\d)',tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#print(tuition_fee)
try:
assessment_en = response.xpath(
'//*[@id="section-4"]/div[2]/div').extract()[0]
assessment_en = remove_tags(assessment_en)
assessment_en = assessment_en.replace('\r\n', '')
assessment_en = assessment_en.replace(' ', '')
assessment_en = assessment_en.replace('\n', '')
assessment_en = "<div><span>" + assessment_en + "</span></div>"
#print(assessment_en)
except:
assessment_en = 'N/A'
#print(assessment_en)
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 1
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 4
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
item["alevel"] = alevel
item["ib"] = ib
item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
item["assessment_en"] = assessment_en
#item["apply_pre"] = ''
yield item
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'Glasgow Caledonian University'
try:
location = response.xpath('//*[@id="Studyoptions"]/div/section/div[2]/div/div/div/section/section/article/div[5]/span').extract()[0]
location = remove_tags(location)
#print(location)
except:
location = 'N/A'
#print(location)
try:
department = response.xpath('//ul[@class="course-department"]').extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
department = department.replace(' ', '')
department = department.replace('\n', '')
department = department.replace('Our Staff', '')
#print(department)
except:
department = ''
#print(department)
try:
degree_name = response.xpath('//*[@id="MainMiddleWrap"]/article/section/header/h2/div/span[2]').extract()[0]
degree_name = remove_tags(degree_name)
degree_name = degree_name.split()[0]
#degree_name = re.findall('(.*)\n.*',degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
degree_name = degree_name.replace(' ','')
#print(degree_name)
except:
degree_name = 'N/A'
#print(degree_name)
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath('//*[@id="MainMiddleWrap"]/article/section/header/h2/div/span[2]').extract()[0]
programme_en = remove_tags(programme_en)
#programme_en = re.findall(' (.*)',programme_en)[0]
programme_en = programme_en.replace(degree_name,'')
programme_en = programme_en.replace(' ','')
#programme_en = programme_en.replace('\n', '')
#programme_en = re.findall((' '),'')[0]
#programme_en = re.findall("(.*)\(.*\)",programme_en)[0]
#programme_en = programme_en.replace('\n','')
#programme_en = programme_en.replace(' ','')
#print(programme_en)
except:
programme_en = 'N/A'
#print(programme_en)
try:
overview_en = response.xpath('//*[@id="MainMiddleWrap"]/article/section/section/section[1]/div/div/div').extract()[0]
overview_en = remove_tags(overview_en)
overview_en = overview_en.replace(' ','')
#overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n','')
overview_en = overview_en.replace('\r\n','')
overview_en = overview_en.replace('\n','')
overview_en = '<div>' + overview_en + '</div>'
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = 'N/A'
#print(overview_en)
try:
start_date = response.xpath('//*[@id="Studyoptions"]/div/section/div[2]/div/div/div/section/section/article/div[4]').extract()[0]
start_date = remove_tags(start_date)
if 'Jan' in start_date:
start_date = '1'
elif 'Sep' in start_date:
start_date = '9'
elif 'Oct' in start_date:
start_date = '10'
else:
start_date = '9'
#print(start_date)
except:
start_date = 'N/A'
#print(start_date)
try:
modules_en = response.xpath('//*[@id="Whatyouwillstudy"]/div/div/div').extract()[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n','\n')
modules_en = modules_en.replace('\r\n','')
modules_en = modules_en.replace(' ','')
modules_en = modules_en.replace(' ','')
modules_en = modules_en.replace('\n','')
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath('//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]').extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ','')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath('//*[@id="Entryrequirements"]/div/div/div').extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = "<div>"+rntry_requirements_en+"</div>"
rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\n', '')
rntry_requirements_en = rntry_requirements_en.replace(' ','')
#rntry_requirements_en =rntry_requirements_en.replace(' ','')
#print(rntry_requirements_en)
except:
rntry_requirements_en = 'N/A'
#print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = '<p>4-year Bachelor\'s Degree (minimum 70%; for MSc Management 60% can be considered) 3-year Junior College plus 2-year undergraduate degree can be considered</p>'
except:
require_chinese_en = ''
try:
ielts_desc = ''
#print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
ielts_b = response.xpath('//*[@id="Entryrequirements"]/div/div/div').extract()[0]
ielts =remove_tags(ielts_b)
ielts = re.findall('IELTS(.*)',ielts_b)[0]
ielts = re.findall('(\d\.\d)',ielts)[0]
print(ielts + "1")
except:
ielts = 'N/A'
#print(ielts)
try:
ielts_l = re.findall('(\d\.\d)', ielts_b)[1]
print(ielts_l + "2")
#ielts_l = remove_tags(ielts_l)
except:
ielts_l = 'N/A'
try:
ielts_s = ielts_l
except:
ielts_s = 0
try:
ielts_r = ielts_l
except:
ielts_r = 0
try:
ielts_w = ielts_l
except:
ielts_w = 0
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = 0
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = 0
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = 0
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = 0
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = 0
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = 0
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = 0
try:
interview_desc_en = response.xpath('//*[@id="entry-requirements-accordion-0"]/div[1]').extract()[0]
interview_desc_en = remove_tags(interview_desc_en)
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
interview_desc_en = "<div>" + interview_desc_en + "</div>"
#print(interview_desc_en)
except:
interview_desc_en = 'N/A'
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath('//*[@id="your-future-career-5"]/div[1]/div').extract()[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\r\n','')
career_en = career_en.replace(' ','')
career_en = career_en.replace('\n','')
career_en = "<div><span>" + career_en + "</span></div>"
#print(career_en)
except:
career_en = ''
#print(career_en)
try:
apply_desc_en = 'Apply Online'
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = '<div>Statement of Purpose Your personal statement is an extremely important aspect of the application. It gives you with the opportunity to share with the university your motivations, background and reasons for applying for this particular course. What we look for in a personal statement Letters of Reference Letters of reference provide the university with additional evidence of your ability to successfully complete your programme of study. Upload or attach your letters of reference to your application, or ask your referees to submit this information directly at [email protected]. What we look for in letters of reference Academic and Professional Certificates Certificates show us what you have already achieved in your previous academic study as well as any CPD during your employment. What we look for in certificates Academic Transcripts Academic transcripts provide the university with full details of the modules you studied at undergraduate level. What we look for in academic transcripts Passport (International Applicants Only) Please provide a photocopy of your passport identification pages.</div>'
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 0
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = response.xpath('//*[@id="Studyoptions"]/div/section/div[2]/div/div/div/section/section/article/div[3]/span').extract()[0]
duration = remove_tags(duration)
#duration = remove_tags(duration)
#duration = re.findall('(\d) Years',duration)[0]
if '36' in duration:
duration = '3'
elif '16' in duration:
duration = '1'
elif '12' in duration:
duration = '1'
elif '3' in duration:
duration = '3'
elif '2' in duration:
duration = '2'
elif '1' in duration:
duration = '1'
#print(duration)
except:
duration = '0'
#print(duration)
try:
other = response.xpath('//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a').extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath('//*[@id="tab-Entry_Requirements"]/div/div[1]/div[1]/table[1]/tbody/tr[11]/td[2]').extract()[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath('//*[@id="tab-Entry_Requirements"]/div/div[1]/div/table[1]').extract()[0]
alevel = remove_tags(alevel)
alevel = re.findall("(\w\w\w) at A Level",alevel)[0]
#print(alevel)
except:
alevel = 'CC'
#print(alevel)
try:
ucascode = response.xpath('/html/body/div[3]/div[1]/div/div/div[2]/div/div[1]/div[1]/div[2]').extract()[0]
ucascode = remove_tags(ucascode)
#print(ucascode)
except:
ucascode = ''
#print(ucascode)
try:
tuition_fee = response.xpath('//*[@id="Feesandfunding"]/div/div/div/p[3]').extract()[0]
tuition_fee = remove_tags(tuition_fee)
tuition_fee = tuition_fee.replace('£','')
tuition_fee = tuition_fee.replace(',','')
tuition_fee = tuition_fee.replace('*','')
tuition_fee = tuition_fee.replace(' ','')
tuition_fee = tuition_fee.replace('\r\n','')
tuition_fee = tuition_fee.replace('\n','')
tuition_fee = re.findall('(\d\d\d\d\d)',tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#print(tuition_fee)
try:
assessment_en = response.xpath('//*[@id="Assessmentmethods"]/div/div/div').extract()[0]
assessment_en = remove_tags(assessment_en)
#assessment_en = assessment_en.replace('\n','')
assessment_en = assessment_en.replace('\r\n','')
assessment_en = assessment_en.replace(' ','')
assessment_en = "<div><span>" + assessment_en + "</span></div>"
#print(assessment_en)
except:
assessment_en = 'N/A'
#print(assessment_en)
try:
teach_time = response.xpath('//*[@id="Studyoptions"]/div/section/div[2]/div/div/div/section/section/article/div[2]').extract()[0]
teach_time = remove_tags(teach_time)
if 'full' in teach_time:
teach_time = 'fulltime'
elif 'Full' in teach_time:
teach_time = 'fulltime'
else:
teach_time = 'parttime'
print(teach_time)
except:
teach_time = 'N/A'
print(teach_time)
teach_type = 'taught'
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 2
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 3
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
#item["alevel"] = alevel
#item["ib"] = ib
#item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
item["assessment_en"] = assessment_en
item["teach_time"] = teach_time
item["teach_type"] = teach_type
#item["apply_pre"] = ''
yield item
def parse(self, response):
# print(response.url)
item = UcasItem()
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'Birkbeck, University of London'
try:
location = 'Central London'
#location = remove_tags(location)
#location = remove_tags(location)
#print(location)
except:
location = 'n/a'
print(location)
try:
department = response.xpath('').extract()
department = remove_tags(department)
except:
department = ''
try:
degree_name = response.xpath('//h1').extract()[0]
degree_name = remove_tags(degree_name)
degree_name = re.findall('\((.*)\)', degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
#print(degree_name)
except:
degree_name = ''
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath('//h1').extract()[0]
programme_en = remove_tags(programme_en)
programme_en = re.findall("(.*)\(.*\)", programme_en)[0]
#programme_en = programme_en.replace('\n','')
#programme_en = programme_en.replace(' ','')
#print(programme_en)
except:
programme_en = ''
#print(programme_en)
try:
overview_en = response.xpath(
'//*[@id="content"]/div[5]').extract()[0]
overview_en = remove_tags(overview_en)
overview_en = '<div>' + overview_en + '</div>'
overview_en = overview_en.replace(' ', '')
overview_en = overview_en.replace('\n\n', '\n')
overview_en = overview_en.replace('\n\n', '\n')
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = ''
try:
start_date = '10'
#print(start_date)
except:
start_date = ''
try:
modules_en = response.xpath(
'//*[@id="courseStructure"]').extract()[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n', '\n')
modules_en = modules_en.replace('\r\n', '')
modules_en = modules_en.replace(' ', '')
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
apply_fee = 0
try:
degree_requirements = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]'
).extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ', '')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath(
'//*[@id="content"]/div[8]/div/div[2]').extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = rntry_requirements_en.replace(' ', '')
rntry_requirements_en = rntry_requirements_en.replace('\n', '')
rntry_requirements_en = re.findall(
'Entry requirements(.*)Visa requirements',
rntry_requirements_en)[0]
#print(rntry_requirements_en)
except:
rntry_requirements_en = ''
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
ielts_desc = response.xpath(
'//*[@id="content"]/div[8]/div/div[2]').extract()[0]
ielts_desc = remove_tags(ielts_desc)
#ielts_desc = re.findall('INTERNATIONAL ENTRY REQUIREMENTS(.*)Visa requirements',ielts_desc)[0]
# print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
ielts = re.findall('(\d\.\d)', ielts_desc)[0]
#i#elts = remove_tags(ielts)
print(ielts)
except:
ielts = '6.5'
try:
ielts_l = re.findall('(\d\.\d)', ielts_desc)[1]
print(ielts_l)
#ielts_l = remove_tags(ielts_l)
except:
ielts_l = '6.0'
try:
ielts_s = ielts_l
except:
ielts_l = ''
try:
ielts_r = ielts_l
except:
ielts_l = ''
try:
ielts_w = ielts_l
except:
ielts_l = ''
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = ''
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = ''
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = ''
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = ''
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = ''
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = ''
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = ''
try:
work_experience_desc_en = response.xpath(
'//*[@id="content"]/div[13]/div/a/div/div[1]/p').extract()[0]
work_experience_desc_en = remove_tags(work_experience_desc_en)
#print(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
interview_desc_en = response.xpath('').extract()
interview_desc_en = remove_tags(interview_desc_en)
except:
interview_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
apply_desc_en = 'If English is not your first language or you have not previously studied in English, our usual requirement is the equivalent of an International English Language Testing System (IELTS Academic Test) score of 6.5, with not less than 6.0 in each of the sub-tests.'
apply_desc_en = remove_tags(apply_desc_en)
apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = '<p>birth certificate or passport IELTS English Language certificate degree certificate or transcript additional documentation requested by Registry. Please note: documentation that may be required by the Admissions Tutor in the relevant school and this should be sent directly to the school.</p>'
apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = '1'
#duration = remove_tags(duration)
#duration = re.findall('(\d) Years',duration)[0]
#print(duration)
except:
duration = ''
#print(duration)
try:
other = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a'
).extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div[1]/table[1]/tbody/tr[11]/td[2]'
).extract()[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div/table[1]'
).extract()[0]
alevel = remove_tags(alevel)
alevel = re.findall("(\w\w\w) at A Level", alevel)[0]
#print(alevel)
except:
alevel = 'CC'
#print(alevel)
try:
ucascode = response.xpath(
'/html/body/div[3]/div[1]/div/div/div[2]/div/div[1]/div[1]/div[2]'
).extract()[0]
ucascode = remove_tags(ucascode)
#print(ucascode)
except:
ucascode = ''
#print(ucascode)
try:
tuition_fee = '12500'
# tuition_fee = remove_tags(tuition_fee)
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
try:
teach_time = response.xpath(
'//*[@id="content"]/div[3]').extract()[0]
teach_time = remove_tags(teach_time)
if 'full-time' in teach_time:
teach_time = 'fulltime'
else:
teach_time = 'parttime'
#print(teach_time)
except:
teach_time = 'N/A'
#print(teach_time)
teach_type = 'taught'
assessment_en = ''
require_chinese_en = '<div>UG A recognised International Foundation Year from a UK institution or a Chinese institution when following a validated UK syllabus. OR Successfully completed first year of a Chinese University degree OR 2 or 3 year Diploma (Zhuanke or Da Zhuan) with a minimum final grade of 70% or equivalent PG Completion of a Bachelor degree from an accredited Chinese university with 75% or higher (GPA 2.9 or above) If you find that your qualifications do not meet our entry requirements, relevant experience and completion of one of BGU’s online pre-arrival courses can also be taken into account to meet the entry criteria.</div>'
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 1
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = ''
item["application_open_date"] = 'all year'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = 0
item["toefl_l"] = 0
item["toefl_s"] = 0
item["toefl_r"] = 0
item["toefl_w"] = 0
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 3
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
#item["alevel"] = alevel
#item["ib"] = ib
#item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
item["teach_time"] = teach_time
item["teach_type"] = teach_type
item["assessment_en"] = assessment_en
#item["apply_pre"] = ''
yield item
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'Heriot-Watt University'
try:
location = response.xpath(
'//*[@id="content-main"]/section[3]/div/div/dl/dd[3]').extract(
)[0]
location = remove_tags(location)
#print(location)
except:
location = 'Edinburgh'
#print(location)
try:
department = response.xpath('').extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
department = department.replace(' ', '')
department = department.replace('\n', '')
department = department.replace('Our Staff', '')
#print(department)
except:
department = ''
#print(department)
try:
degree_name = response.xpath(
'//*[@id="content-main"]/section[1]/div/div[2]/div[2]/div/h1'
).extract()[0]
degree_name = remove_tags(degree_name)
degree_name = re.findall('.*,(.*)', degree_name)[0]
#degree_name = re.findall('\r\n',degree_name)
#degree_name = re.findall('(.*)\n.*',degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
degree_name = degree_name.replace(' ', '')
degree_name = degree_name.replace('\r\n', '')
degree_name = degree_name.replace('\n', '')
#print(degree_name)
except:
degree_name = 'N/A'
#print(degree_name)
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath(
'//*[@id="content-main"]/section[1]/div/div[2]/div[2]/div/h1'
).extract()[0]
programme_en = remove_tags(programme_en)
programme_en = programme_en.replace('\r\n', '')
#programme_en = re.findall('',programme_en)[0]
programme_en = programme_en.replace(' ', ' ')
#programme_en = programme_en.replace(degree_name,'')
#programme_en = programme_en.replace('()','')
#print(programme_en)
except:
programme_en = 'N/A'
#print(programme_en)
try:
overview_en = response.xpath(
'//*[@id="content-main"]/section[6]/div/div').extract()[0]
overview_en = remove_tags(overview_en)
overview_en = overview_en.replace(' ', ' ')
#overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n', '')
overview_en = overview_en.replace('\r\n', '')
overview_en = overview_en.replace('\n', '')
overview_en = '<div>' + overview_en + '</div>'
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = 'N/A'
#print(overview_en)
try:
start_date = response.xpath('//*[@id="course-tab-5"]').extract()[0]
start_date = remove_tags(start_date)
start_date = start_date.replace('\r\n', '')
start_date = start_date.replace(' ', ' ')
start_date = start_date.replace('\n', '')
start_date = re.findall('Start Date(.*)', start_date)[0]
if 'October' in start_date:
start_date = '2019-10'
elif '24' in start_date:
start_date = '2019-9-24'
else:
start_date = '2019-9'
#print(start_date)
except:
start_date = 'N/A'
#print(start_date)
try:
modules_en = response.xpath(
'//*[@id="course-content"]').extract()[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n', '\n')
modules_en = modules_en.replace('\r\n', '')
modules_en = modules_en.replace(' ', ' ')
#modules_en = modules_en.replace(' ','')
modules_en = modules_en.replace('\n', '')
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]'
).extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ', '')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath(
'//*[@id="entry-requirements"]').extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = "<div>" + rntry_requirements_en + "</div>"
rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\n', '')
rntry_requirements_en = rntry_requirements_en.replace(' ', ' ')
#rntry_requirements_en =rntry_requirements_en.replace(' ','')
#print(rntry_requirements_en)
except:
rntry_requirements_en = 'N/A'
#print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = ''
except:
require_chinese_en = ''
try:
ielts_desc = ''
#print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
aa = response.xpath('//*[@id="entry-requirements"]').extract()[0]
aa = remove_tags(aa)
ielts = re.findall('\d\.\d', aa)[0]
#ielts = 0
#print(ielts)
except:
ielts = 0
#print(ielts)
try:
#ielts_l = '5.5'
ielts_l = re.findall('\d\.\d', aa)[1]
#print(ielts_l)
#ielts_l = remove_tags(ielts_l)
except:
ielts_l = 0
#print(ielts)
try:
ielts_s = ielts_l
except:
ielts_s = ielts_l
try:
ielts_r = ielts_l
except:
ielts_r = ielts_l
try:
ielts_w = ielts_l
except:
ielts_w = ielts_l
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = 0
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = 0
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = 0
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = 0
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = 0
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = 0
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = 0
try:
interview_desc_en = response.xpath(
'//*[@id="entry-requirements-accordion-0"]/div[1]').extract(
)[0]
interview_desc_en = remove_tags(interview_desc_en)
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
interview_desc_en = "<div>" + interview_desc_en + "</div>"
#print(interview_desc_en)
except:
interview_desc_en = 'N/A'
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath('//*[@id="career"]').extract()[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\r\n', '')
career_en = career_en.replace(' ', ' ')
career_en = career_en.replace('\n', '')
career_en = "<div><span>" + career_en + "</span></div>"
#print(career_en)
except:
career_en = ''
#print(career_en)
try:
apply_desc_en = ''
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = ''
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 0
dead_time = '1-15'
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = response.xpath('//dl/dd[2]').extract()[0]
#duration = remove_tags(duration)
duration = remove_tags(duration)
duration = duration.replace(' ', '')
duration = duration.replace('\r\n', '')
duration = duration.replace('\n', '')
#duration = re.findall('Duration(.*)',duration)[0]
#duration = re.findall('(\d) Years',duration)[0]
if '4' in duration:
duration = '4'
elif '3' in duration:
duration = '3'
elif '5' in duration:
duration = '5'
elif '2' in duration:
duration = '2'
elif '1' in duration:
duration = '1'
elif '6' in duration:
duration = '6'
else:
duration = 'N/A'
#print(duration)
except:
duration = 'N/A'
#print(duration)
try:
other = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a'
).extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath(
'//*[@id="entry-requirements"]/ul[1]').extract()[0]
ib = remove_tags(ib)
ib = re.findall('Int. Baccalaureate(.*)', ib)[0]
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath(
'//*[@id="entry-requirements"]/ul[1]/li[2]').extract()[0]
alevel = remove_tags(alevel)
#alevel = re.findall('entry, (.*), IB',alevel)[0]
#alevel = alevel.replace('*','')
#alevel = re.findall("(\w\w\w)",alevel)[0]
#print(alevel)
except:
alevel = 'N/A'
#print(alevel)
try:
ucascode = response.xpath(
'//*[@id="content-main"]/section[3]/div/div/dl/dd[1]').extract(
)[0]
ucascode = remove_tags(ucascode)
ucascode = ucascode.replace('\r\n', '')
ucascode = ucascode.replace('\n', '')
ucascode = ucascode.replace(' ', ' ')
#ucascode = re.findall('UCAS Code(.*)Award',ucascode)[0]
#ucascode = ucascode.replace(' ','')
#ucascode = ucascode.replace(' ','')
#print(ucascode)
except:
ucascode = 'N/A'
#print(ucascode)
try:
tuition_fee = response.xpath(
'//*[@id="fees-and-funding"]/table/tbody/tr/td[3]').extract(
)[0]
tuition_fee = remove_tags(tuition_fee)
tuition_fee = tuition_fee.replace('£', '')
tuition_fee = tuition_fee.replace(',', '')
tuition_fee = tuition_fee.replace('*', '')
tuition_fee = tuition_fee.replace(' ', '')
tuition_fee = tuition_fee.replace('\r\n', '')
tuition_fee = tuition_fee.replace('\n', '')
tuition_fee = re.findall('(\d\d\d\d\d)', tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#print(tuition_fee)
try:
assessment_en = response.xpath(
'//*[@id="assessment-methods"]/div/p').extract()[0]
assessment_en = remove_tags(assessment_en)
assessment_en = assessment_en.replace('\r\n', '')
assessment_en = assessment_en.replace(' ', '')
assessment_en = assessment_en.replace('\n', '')
assessment_en = "<div><span>" + assessment_en + "</span></div>"
#print(assessment_en)
except:
assessment_en = ''
#print(assessment_en)
application_open_date = '2018-10-6/2018-10-20'
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 1
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 3
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
item["alevel"] = alevel
item["ib"] = ib
item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
item["assessment_en"] = assessment_en
item["application_open_date"] = application_open_date
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'Bishop Grosseteste University'
try:
location = 'Dundee'
#location = remove_tags(location)
#print(location)
except:
location = 'N/A'
#print(location)
try:
department = response.xpath(
'//*[@id="bgu-single"]/div/article/div/div[1]/div[2]/div/table/tbody/tr[3]/td[2]'
).extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
department = department.replace(' ', '')
department = department.replace('\n', '')
#department = department.replace('Our Staff', '')
#print(department)
except:
department = ''
#print(department)
try:
degree_name = response.xpath(
'//*[@id="main-content"]/div[1]/div/div/div[2]/h1').extract(
)[0]
degree_name = remove_tags(degree_name)
#degree_name = degree_name.split()[-1]
degree_name = re.findall('\((.*)\).*', degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
#degree_name = degree_name.replace(' ','')
#print(degree_name)
except:
degree_name = 'N/A'
#print(degree_name)
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath(
'//*[@id="main-content"]/div[1]/div/div/div[2]/h1').extract(
)[0]
programme_en = remove_tags(programme_en)
#programme_en = programme_en.split()[1]
#programme_en = re.findall(' (.*)',programme_en)[0]
#programme_en = programme_en.replace(degree_name,'')
#programme_en = programme_en.replace(' ','')
#programme_en = programme_en.replace('\n', '')
#programme_en = re.findall((' '),'')[0]
#programme_en = re.findall("\(.*\)(.*)",programme_en)[0]
#programme_en = programme_en.replace('\n','')
#programme_en = programme_en.replace(' ','')
#print(programme_en)
except:
programme_en = 'N/A'
#print(programme_en)
try:
overview_en = response.xpath(
'//*[@id="bgu-single"]/div/article/div/div[1]/div[1]/div/p'
).extract()[0]
overview_en = remove_tags(overview_en)
#overview_en = overview_en.replace(' ','')
#overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n', '')
overview_en = overview_en.replace('\r\n', '')
overview_en = overview_en.replace('\n', '')
overview_en = '<div>' + overview_en + '</div>'
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = 'N/A'
#print(overview_en)
try:
start_date = '9,10'
#print(start_date)
except:
start_date = ''
try:
modules_en = response.xpath(
'//*[@id="bgu-single"]/div/article/div/div[4]/div/div'
).extract()[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n', '\n')
modules_en = modules_en.replace('\r\n', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace('\n', '')
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]'
).extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ', '')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath(
'//*[@id="bgu-single"]/div/article/div/div[3]/div/div'
).extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = "<div>" + rntry_requirements_en + "</div>"
rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\n', '')
rntry_requirements_en = rntry_requirements_en.replace(' ', '')
#rntry_requirements_en =rntry_requirements_en.replace(' ','')
print(rntry_requirements_en)
except:
rntry_requirements_en = 'N/A'
print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = ''
except:
require_chinese_en = ''
try:
ielts_desc = response.xpath(
'//*[@id="entry-requirements"]/div/section[2]/div/ul/li'
).extract()[0]
ielts_desc = remove_tags(ielts_desc)
#print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
#ielts = '6.5'
#ielts =remove_tags(ielts)
#ielts = re.findall('IELTS(.*)',ielts)[0]
ielts = '6.5'
#print(ielts)
except:
ielts = 0
#print(ielts)
try:
#ielts_l = '5.5'
ielts_l = '6.0'
#print(ielts_l)
#ielts_l = remove_tags(ielts_l)
except:
ielts_l = 0
try:
ielts_s = ielts_l
except:
ielts_s = ielts_l
try:
ielts_r = ielts_l
except:
ielts_r = ielts_l
try:
ielts_w = ielts_l
except:
ielts_w = ielts_l
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = 0
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = 0
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = 0
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = 0
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = 0
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = 0
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = 0
try:
interview_desc_en = response.xpath(
'//*[@id="entry-requirements-accordion-0"]/div[1]').extract(
)[0]
interview_desc_en = remove_tags(interview_desc_en)
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
interview_desc_en = "<div>" + interview_desc_en + "</div>"
#print(interview_desc_en)
except:
interview_desc_en = 'N/A'
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath(
'//div[@class = "cc-careers mrt_30"]').extract()[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\r\n', '')
career_en = career_en.replace(' ', '')
career_en = career_en.replace('\n', '')
career_en = "<div><span>" + career_en + "</span></div>"
career_en = career_en.replace('&', '')
#print(career_en)
except:
career_en = ''
#print(career_en)
try:
apply_desc_en = '<div>Starting your university application can be a complicated process, but don\'t worry - we’re here to help every step of the way. The easiest way to apply is to search below and select your course, then simply click the \'APPLY NOW\' button on the course page.</div>'
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = ''
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 12
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = 1
#duration = remove_tags(duration)
#duration = remove_tags(duration)
#duration = re.findall('(\d) Years',duration)[0]
# if '36' in duration:
# duration = '3'
# elif '16' in duration:
# duration = '1'
# elif '12' in duration:
# duration = '1'
# elif '3' in duration:
# duration = '3'
# elif '2' in duration:
# duration = '2'
# elif '1' in duration:
# duration = '1'
# elif 'two' in duration:
# duration = '2'
# else:
# duration = '1'
# #print(duration)
except:
duration = 0
#print(duration)
try:
other = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a'
).extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath(
'//*[@id="bgu-single"]/div/article/div/div[3]/div/div'
).extract()[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div/table[1]'
).extract()[0]
alevel = remove_tags(alevel)
alevel = re.findall("(\w\w\w) at A Level", alevel)[0]
#print(alevel)
except:
alevel = 'N/A'
#print(alevel)
try:
ucascode = response.xpath(
'//*[@id="bgu-single"]/div/article/div/div[1]/div[2]/div/table/tbody/tr[2]/td[2]'
).extract()[0]
ucascode = remove_tags(ucascode)
#print(ucascode)
except:
ucascode = ''
#print(ucascode)
try:
tuition_fee = response.xpath(
'/html/body/div[1]/div/div[1]/section[7]/div/div/div[3]/div/div[2]'
).extract()[0]
tuition_fee = remove_tags(tuition_fee)
tuition_fee = tuition_fee.replace('£', '')
tuition_fee = tuition_fee.replace(',', '')
tuition_fee = tuition_fee.replace('*', '')
tuition_fee = tuition_fee.replace(' ', '')
tuition_fee = tuition_fee.replace('\r\n', '')
tuition_fee = tuition_fee.replace('\n', '')
tuition_fee = re.findall('(\d\d\d\d\d)', tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#print(tuition_fee)
try:
assessment_en = response.xpath(
'//*[@id="bgu-single"]/div/article/div/div[7]').extract()[0]
assessment_en = remove_tags(assessment_en)
assessment_en = assessment_en.replace(' ', ' ')
assessment_en = assessment_en.replace('\r\n', '')
assessment_en = assessment_en.replace('\n', '')
assessment_en = assessment_en.replace(
' ', '')
assessment_en = assessment_en.replace(
' ',
'')
assessment_en = '<div>' + assessment_en + '</div>'
#print(assessment_en)
except:
assessment_en = 'N/A'
#print(assessment_en)
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 2
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 3
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
item["alevel"] = alevel
item["ib"] = ib
item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
#item["require_chinese_en"] = require_chinese_en
item["assessment_en"] = assessment_en
#item["apply_pre"] = ''
yield item
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'York St John University'
try:
location = response.xpath(
'//*[@id="section-overview"]/div/div[2]/dl[1]/dd').extract()[0]
location = remove_tags(location)
#print(location)
except:
location = 'N/A'
#print(location)
try:
department = response.xpath(
'/html/body/div[1]/div/div/div[2]/div[1]/div[2]/p[2]/a[3]/strong'
).extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
department = department.replace(' ', '')
department = department.replace('\n', '')
department = department.replace('Our Staff', '')
#print(department)
except:
department = ''
#print(department)
try:
degree_name = response.xpath(
'/html/body/main/article/div[1]/header/div/h1/em').extract()[0]
degree_name = remove_tags(degree_name)
degree_name = degree_name.split()[0]
#degree_name = re.findall('(.*)\n.*',degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
degree_name = degree_name.replace(' ', '')
#print(degree_name)
except:
degree_name = 'N/A'
#print(degree_name)
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en,
keep=('div', 'p', 'ul', 'li',
'span'))
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath(
'//html/body/main/article/div[1]/header/div/h1/text()'
).extract()[0]
programme_en = remove_tags(programme_en)
#programme_en = re.findall(' (.*)',programme_en)[0]
programme_en = programme_en.replace(degree_name, '')
programme_en = programme_en.replace(' ', '')
#programme_en = programme_en.replace('\n', '')
#programme_en = re.findall((' '),'')[0]
#programme_en = re.findall("(.*)\(.*\)",programme_en)[0]
#programme_en = programme_en.replace('\n','')
#programme_en = programme_en.replace(' ','')
#print(programme_en)
except:
programme_en = 'N/A'
#print(programme_en)
try:
overview_en = response.xpath(
'//*[@id="section-overview"]/div/div[1]').extract()[0]
overview_en = remove_tags(overview_en,
keep=('div', 'p', 'ul', 'li', 'span'))
overview_en = overview_en.replace(' ', '')
#overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n', '')
overview_en = overview_en.replace('\r\n', '')
overview_en = overview_en.replace('\n', '')
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = 'N/A'
#print(overview_en)
try:
start_date = response.xpath(
'//*[@id="section-overview"]/div/div[2]').extract()[0]
start_date = remove_tags(start_date)
if 'January' in start_date:
start_date = '1'
else:
start_date = '9'
print(start_date)
except:
start_date = '9'
print(start_date)
try:
modules_en = response.xpath('//*[@id="level-1"]/div').extract()[0]
modules_en = remove_tags(modules_en,
keep=('div', 'p', 'ul', 'li', 'span'))
modules_en = modules_en.replace('\n\n', '\n')
modules_en = modules_en.replace('\r\n', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace('\n', '')
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]'
).extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ', '')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath(
'//*[@id="reqs-1"]/div').extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = "<div>" + rntry_requirements_en + "</div>"
rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\n', '')
rntry_requirements_en = rntry_requirements_en.replace(' ', '')
#rntry_requirements_en =rntry_requirements_en.replace(' ','')
#print(rntry_requirements_en)
except:
rntry_requirements_en = 'N/A'
#print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = ''
except:
require_chinese_en = ''
try:
ielts_desc = response.xpath('//*[@id="reqs-2"]/div/p').extract()[0]
ielts_desc = remove_tags(ielts_desc)
#print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
ielts = response.xpath('//*[@id="reqs-2"]/div').extract()[0]
ielts = remove_tags(ielts)
#ielts = re.findall('IELTS(.*)',ielts)[0]
ielts = re.findall('(\d\.\d)', ielts)[0]
#print(ielts)
except:
ielts = 0
#print(ielts)
try:
#ielts_l = re.findall('IELTS(.*)',ielts)[1]
ielts_l = response.xpath('//*[@id="reqs-2"]/div').extract()[0]
ielts_l = re.findall('(\d\.\d)', ielts_l)[1]
#print(ielts_l)
#ielts_l = remove_tags(ielts_l)
except:
ielts_l = 0
try:
ielts_s = ielts_l
except:
ielts_s = ielts_l
try:
ielts_r = ielts_l
except:
ielts_r = ielts_l
try:
ielts_w = ielts_l
except:
ielts_w = ielts_l
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = 0
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = 0
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = 0
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = 0
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = 0
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = 0
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = 0
try:
interview_desc_en = response.xpath(
'//*[@id="entry-requirements-accordion-0"]/div[1]').extract(
)[0]
interview_desc_en = remove_tags(interview_desc_en,
keep=('div', 'p', 'ul', 'li',
'span'))
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
#print(interview_desc_en)
except:
interview_desc_en = 'N/A'
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath(
'//*[@id="collapseCareerOpportunities"]/div').extract()[0]
career_en = remove_tags(career_en,
keep=('div', 'p', 'ul', 'li', 'span'))
career_en = career_en.replace('\r\n', '')
career_en = career_en.replace(' ', '')
career_en = career_en.replace('\n', '')
#print(career_en)
except:
career_en = ''
#print(career_en)
try:
apply_desc_en = '<div>You can apply directly to the course via our ‘Apply Now’ links. Please select the variant of the course that you intend to undertake (e.g. full-time or part-time) as the link will take you to a customised form for the specific course. You will need to create a login and password and complete the online form. Please contact two referees in advance of submitting your application as an automated request will go out as soon as you submit, and your application will not be reviewed until both references are in place. Applications for September 2017 entry must be submitted and completed by 6 October 2017.</div>'
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = ''
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 0
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = response.xpath(
'//*[@id="section-overview"]/div/div[2]/dl[3]/dd').extract()[0]
duration = remove_tags(duration)
#duration = remove_tags(duration)
#duration = re.findall('(\d) Years',duration)[0]
if '1' in duration:
duration = '1'
elif '2' in duration:
duration = '2'
else:
duration = '0'
#print(duration)
except:
duration = 0
#print(duration)
try:
other = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a'
).extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div[1]/table[1]/tbody/tr[11]/td[2]'
).extract()[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div/table[1]'
).extract()[0]
alevel = remove_tags(alevel)
alevel = re.findall("(\w\w\w) at A Level", alevel)[0]
#print(alevel)
except:
alevel = 'CC'
#print(alevel)
try:
ucascode = response.xpath(
'/html/body/div[3]/div[1]/div/div/div[2]/div/div[1]/div[1]/div[2]'
).extract()[0]
ucascode = remove_tags(ucascode)
#print(ucascode)
except:
ucascode = ''
#print(ucascode)
try:
tuition_fee = response.xpath(
'///*[@id="section-overview"]/div/div[2]').extract()[0]
tuition_fee = remove_tags(tuition_fee)
tuition_fee = tuition_fee.replace('£', '')
tuition_fee = tuition_fee.replace(',', '')
tuition_fee = tuition_fee.replace('*', '')
tuition_fee = tuition_fee.replace(' ', '')
tuition_fee = tuition_fee.replace('\r\n', '')
tuition_fee = tuition_fee.replace('\n', '')
tuition_fee = re.findall('(\d\d\d\d\d)', tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#print(tuition_fee)
try:
teach_time = response.xpath(
'//*[@id="section-overview"]/div/div[2]').extract()[0]
teach_time = remove_tags(teach_time)
if 'full' in teach_time:
teach_time = 'fulltime'
elif 'Full' in teach_time:
teach_time = 'fulltime'
else:
teach_time = 'parttime'
print(teach_time)
except:
teach_time = 'N/A'
print(teach_time)
teach_type = 'taught'
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 2
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 2
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
#item["alevel"] = alevel
#item["ib"] = ib
#item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
#item["apply_pre"] = ''
item["teach_time"] = teach_time
item["teach_type"] = teach_type
item["assessment_en"] = ''
yield item
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'Sussex University'
try:
location = 'Brighton'
#location = remove_tags(location)
#print(location)
except:
location = None
#print(location)
try:
department = response.xpath('').extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
department = department.replace(' ', '')
department = department.replace('\n', '')
#department = department.replace('Our Staff', '')
#print(department)
except:
department = None
#print(department)
try:
degree_name = response.xpath(
'//*[@id="main"]/div[2]/h1').extract()[0]
degree_name = remove_tags(degree_name)
#degree_name = degree_name.split()[0]
#degree_name = re.findall('\((.*)\).*',degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
#degree_name = degree_name.replace(' ','')
degree_name = re.findall('.*([A-Z][A-Z][A-Z].*)', degree_name)[0]
# print(degree_name)
except:
degree_name = re.findall('.*([A-Z][A-Z].*)', degree_name)[0]
# print(degree_name)
try:
degree_overview_en = response.xpath(
'//h2[contains(text(),"Key information")]/following-sibling::p'
).extract()
degree_overview_en = ''.join(degree_overview_en)
degree_overview_en = re.sub(' [a-zA-Z\-]*=[\'\"].+?[\'\"]', '',
degree_overview_en)
#print(degree_overview_en)
except:
degree_overview_en = None
#print(degree_overview_en)
try:
programme_en = response.xpath(
'//*[@id="main"]/div[2]/h1').extract()[0]
programme_en = remove_tags(programme_en)
programme_en = programme_en.replace(degree_name, '')
programme_en = programme_en.rstrip(' ')
#programme_en = programme_en.split()[1]
#programme_en = re.findall(' (.*)',programme_en)[0]
#programme_en = programme_en.replace(degree_name,'')
#programme_en = programme_en.replace(' ','')
#programme_en = programme_en.replace('\n', '')
#programme_en = re.findall((' '),'')[0]
#programme_en = re.findall("\(.*\)(.*)",programme_en)[0]
#programme_en = programme_en.replace('\n','')
#programme_en = programme_en.replace(' ','')
# print(programme_en)
except:
programme_en = None
# print(programme_en)
try:
overview_en = degree_overview_en
# overview_en = remove_tags(overview_en)
#overview_en = re.findall('COURSE OVERVIEW(.*)',overview_en)[0]
# overview_en = overview_en.replace(' ','')
# overview_en = overview_en.replace('\n\n','\n')
# overview_en = overview_en.replace('\n\n','')
# overview_en = overview_en.replace('\r\n','')
# overview_en = overview_en.replace('\n','')
# overview_en = re.findall('COURSE OVERVIEW(.*)Careers',overview_en)[0]
#overview_en = '<div>' + overview_en + '</div>'
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = None
#print(overview_en)
try:
start_date = response.xpath(
'//dt[contains(text(),"Start date:")]/following-sibling::dd[1]'
).extract()[0]
start_date = remove_tags(start_date)
if 'September 2019' in start_date:
start_date = '2019-09'
else:
start_date = None
# print(start_date)
except:
start_date = None
# print(start_date)
try:
#modules_en = response.xpath('//div[4]/div/div/div[1]/div[5]/div/div[2]/p').extract()[0]
modules_en = response.xpath(
'//h4[contains(text(),"Core modules")]/following-sibling::*'
).extract()
modules_en = ''.join(modules_en)
modules_en = re.sub(' [a-zA-Z\-]*=[\'\"].+?[\'\"]', '', modules_en)
# overview_en = re.findall('COURSE OVERVIEW(.*)',overview_en)[0]
# modules_en = modules_en.replace(' ', '')
# modules_en = modules_en.replace('\n\n', '\n')
#modules_en = modules_en.replace('\n\n', '')
# modules_en = modules_en.replace('\r\n', '')
# modules_en = modules_en.replace('\n', '')
# modules_en = re.findall('Year 1(.*)in Year 1', modules_en)[0]
# modules_en = '<div>' + modules_en + '</div>'
# print(modules_en)
except:
modules_en = None
# print(modules_en)
# try:
#degree_requirements = require_chinese_en
# degree_requirements = remove_tags(degree_requirements)
#degree_requirements = degree_requirements.replace(' ','')
#print(degree_requirements)
# except:
# d#egree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath(
'//*[@id="tab-content-uk"]').extract()
rntry_requirements_en = ''.join(rntry_requirements_en)
rntry_requirements_en = re.sub(' [a-zA-Z\-]*=[\'\"].+?[\'\"]', '',
rntry_requirements_en)
# rntry_requirements_en = remove_tags(rntry_requirements_en)
# rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
# rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
#rntry_requirements_en = rntry_requirements_en.replace('\n', '')
# rntry_requirements_en = rntry_requirements_en.replace(' ','')
# rntry_requirements_en = re.findall('ENTRY REQUIREMENTS(.*)Visit us',rntry_requirements_en)[0]
#rntry_requirements_en = "<div>"+rntry_requirements_en+"</div>"
#rntry_requirements_en =rntry_requirements_en.replace(' ','')
# print(rntry_requirements_en)
except:
rntry_requirements_en = None
# print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = response.xpath(
'//h4[contains(text(),"China")]/following-sibling::table'
).extract()
require_chinese_en = ''.join(require_chinese_en)
require_chinese_en = re.sub(' [a-zA-Z\-]*=[\'\"].+?[\'\"]', '',
require_chinese_en)
# print(require_chinese_en)
except:
require_chinese_en = None
#print(require_chinese_en)
try:
ielts_desc = response.xpath(
'//h4[contains(text(),"IELTS")]/following-sibling::p[1]'
).extract()[0]
ielts_desc = remove_tags(ielts_desc)
# print(ielts_desc)
except:
ielts_desc = None
#print(ielts_desc)
#print(ielts_desc)
try:
#ielts = '6.5'
#ielts =remove_tags(ielts)
ielts = re.findall('(\d\.\d)', ielts_desc)[0]
#ielts =
# print(ielts)
except:
ielts = None
#print(ielts)
try:
# ielts_l = '5.5'
ielts_l = re.findall('(\d\.\d)', ielts_desc)[1]
#ielts =
# print(ielts_l)
#ielts_l = remove_tags(ielts_l)
except:
ielts_l = None
# print(ielts_l)
try:
ielts_s = ielts_l
except:
ielts_s = ielts_l
try:
ielts_r = ielts_l
except:
ielts_r = ielts_l
try:
ielts_w = ielts_l
except:
ielts_w = ielts_l
try:
toefl_code = '9166'
#toefl_code = remove_tags(toefl_code)
except:
toefl_code = None
try:
toefl_desc = response.xpath(
'//h5[contains(text(),"TOEFL")]/following-sibling::p[1]'
).extract()
toefl_desc = ''.join(toefl_desc)
toefl_desc = remove_tags(toefl_desc)
# print(toefl_desc)
except:
toefl_desc = None
# print(toefl_desc)
try:
toefl = re.findall('\d\d', toefl_desc)[0]
# toefl = remove_tags(toefl)
#print(toefl)
except:
toefl = None
# print(toefl)
try:
toefl_l = re.findall('\d\d', toefl_desc)[1]
toefl_l = remove_tags(toefl_l)
except:
toefl_l = None
try:
toefl_s = re.findall('\d\d', toefl_desc)[3]
# toefl_s = remove_tags(toefl_s)
except:
toefl_s = None
try:
toefl_r = re.findall('\d\d', toefl_desc)[2]
# toefl_r = remove_tags(toefl_r)
except:
toefl_r = None
try:
toefl_w = re.findall('\d\d', toefl_desc)[4]
# toefl_w = remove_tags(toefl_w)
# print(toefl_w)
except:
toefl_w = None
# print(toefl_w)
try:
interview_desc_en = response.xpath(
'//*[@id="entry-requirements-accordion-0"]/div[1]').extract(
)[0]
interview_desc_en = remove_tags(interview_desc_en)
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
interview_desc_en = "<div>" + interview_desc_en + "</div>"
#print(interview_desc_en)
except:
interview_desc_en = None
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = None
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = None
try:
career_en = response.xpath(
'//h2[contains(text(),"Careers")]/following-sibling::*'
).extract()
career_en = ''.join(career_en)
career_en = re.sub(' [a-zA-Z\-]*=[\'\"].+?[\'\"]', '', career_en)
# career_en = career_en.replace(' ','')
# career_en = career_en.replace('\n','')
# career_en = "<div><span>" + career_en + "</span></div>"
# print(career_en)
except:
career_en = None
# print(career_en)
try:
apply_desc_en = '<div>Most applications for Masters courses are made directly to Sussex through the postgraduate application system.https://www.sussex.ac.uk/study/masters/apply/log-into-account</div>'
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = None
try:
apply_documents_en = None
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = None
apply_fee = 0
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = 1
#duration = remove_tags(duration)
#duration = remove_tags(duration)
#duration = re.findall('(\d) Years',duration)[0]
# if '36' in duration:
# duration = '3'
# elif '16' in duration:
# duration = '1'
# elif '12' in duration:
# duration = '1'
# elif '3' in duration:
# duration = '3'
# elif '2' in duration:
# duration = '2'
# elif '1' in duration:
# duration = '1'
# elif 'two' in duration:
# duration = '2'
# else:
# duration = '1'
# #print(duration)
except:
duration = 0
#print(duration)
try:
other = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a'
).extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
tuition_fee = response.xpath(
'//dt[contains(text(),"International students")]/following-sibling::dd'
).extract()[0]
tuition_fee = remove_tags(tuition_fee)
tuition_fee = tuition_fee.replace(',',
'').replace(' per year',
'').replace('£', '')
# tuition_fee = tuition_fee.replace('£','')
# tuition_fee = tuition_fee.replace(',','')
# tuition_fee = tuition_fee.replace('*','')
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\r\n','')
# tuition_fee = tuition_fee.replace('\n','')
#
# tuition_fee = re.findall('(\d\d\d\d\d)',tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
print(tuition_fee)
except:
tuition_fee = None
print(tuition_fee)
item["assessment_en"] = None
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 2
item["degree_name"] = degree_name
# item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = require_chinese_en
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 22
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
#item["alevel"] = alevel
#item["ib"] = ib
#item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
#item["apply_pre"] = ''
yield item
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'Arts University Bournemouth'
try:
location = 'Woodlane, Falmouth'
location = remove_tags(location)
#print(location)
except:
location = 'N/A'
#print(location)
try:
department = response.xpath('/html/body/div[1]/div/div/div[2]/div[1]/div[2]/p[2]/a[3]/strong').extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
department = department.replace(' ', '')
department = department.replace('\n', '')
department = department.replace('Our Staff', '')
#print(department)
except:
department = ''
#print(department)
try:
degree_name = 'BA(Hons)'
#degree_name = degree_name.split()[-1]
#degree_name = re.findall('(.*)\n.*',degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
degree_name = degree_name.replace(' ','')
#print(degree_name)
except:
degree_name = 'N/A'
#print(degree_name)
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath('//*[@id="content"]/main/div/div[1]/article/header/h1').extract()[0]
programme_en = remove_tags(programme_en)
#programme_en = re.findall(' (.*)',programme_en)[0]
#programme_en = programme_en.replace(degree_name,'')
programme_en = programme_en.replace(' ','')
#programme_en = programme_en.replace('\n', '')
#programme_en = re.findall((' '),'')[0]
#programme_en = re.findall("(.*)\(.*\)",programme_en)[0]
#programme_en = programme_en.replace('\n','')
#programme_en = programme_en.replace(' ','')
#print(programme_en)
except:
programme_en = 'N/A'
#print(programme_en)
try:
overview_en = response.xpath('//*[@id="overview"]').extract()[0]
overview_en = remove_tags(overview_en)
overview_en = overview_en.replace(' ','')
#overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n','')
overview_en = overview_en.replace('\r\n','')
overview_en = overview_en.replace('\n','')
overview_en = '<div>' + overview_en + '</div>'
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = 'N/A'
#print(overview_en)
try:
start_date = '9'
#print(start_date)
except:
start_date = ''
try:
modules_en = response.xpath('//*[@id="study"]/div[2]/div').extract()[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n','\n')
modules_en = modules_en.replace('\r\n','')
modules_en = modules_en.replace(' ','')
modules_en = modules_en.replace(' ','')
modules_en = modules_en.replace('\n','')
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath('//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]').extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ','')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
#
# try:
# #rntry_requirements_en = ''
# #rntry_requirements_en = remove_tags(rntry_requirements_en)
# # rntry_requirements_en = "<div>"+rntry_requirements_en+"</div>"
# # rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
# # rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
# # rntry_requirements_en = rntry_requirements_en.replace('\n', '')
# # rntry_requirements_en = rntry_requirements_en.replace(' ','')
# # #rntry_requirements_en =rntry_requirements_en.replace(' ','')
# #print(rntry_requirements_en)
# except:
# rntry_requirements_en = 'N/A'
# #print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = ''
except:
require_chinese_en = ''
try:
ielts_desc = 'IELTS score of 6.5 or above'
#print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
ielts = '6.5'
#ielts =remove_tags(ielts)
#ielts = re.findall('IELTS(.*)',ielts)[0]
#ielts = re.findall('(\d\.\d)',ielts)[0]
#print(ielts)
except:
ielts = 6.5
#print(ielts)
try:
ielts_l = 6.0
#print(ielts_l)
ielts_l = remove_tags(ielts_l)
except:
ielts_l = 6.0
try:
ielts_s = 6.0
except:
ielts_s = 6.0
try:
ielts_r = 6.0
except:
ielts_r = 6.0
try:
ielts_w = 6.0
except:
ielts_w = 6.0
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = 0
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = 0
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = 0
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = 0
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = 0
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = 0
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = 0
try:
interview_desc_en = response.xpath('//*[@id="entry-requirements-accordion-0"]/div[1]').extract()[0]
interview_desc_en = remove_tags(interview_desc_en)
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
interview_desc_en = "<div>" + interview_desc_en + "</div>"
#print(interview_desc_en)
except:
interview_desc_en = 'N/A'
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath('//*[@id="collapseCareerOpportunities"]/div').extract()[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\r\n','')
career_en = career_en.replace(' ','')
career_en = career_en.replace('\n','')
career_en = "<div><span>" + career_en + "</span></div>"
#print(career_en)
except:
career_en = ''
#print(career_en)
try:
apply_desc_en = '<p>You can apply from late October and there is no set deadline for applications. It’s worth remembering that overseas applicants who pay one term’s fees in advance by 1st August will receive a 10% discount. We encourage you to apply early, so that you have time to organise accommodation and to arrange for a visa, if required, before you enrol.</p>'
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = ''
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 0
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = '1'
except:
duration = '0'
#print(duration)
try:
other = response.xpath('//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a').extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath('//*[@id="tab-Entry_Requirements"]/div/div[1]/div[1]/table[1]/tbody/tr[11]/td[2]').extract()[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath('//*[@id="tab-Entry_Requirements"]/div/div[1]/div/table[1]').extract()[0]
alevel = remove_tags(alevel)
alevel = re.findall("(\w\w\w) at A Level",alevel)[0]
#print(alevel)
except:
alevel = 'N/A'
#print(alevel)
try:
ucascode = response.xpath('//*[@id="overview"]/div/div/div[2]/span[2]').extract()[0]
ucascode = remove_tags(ucascode)
ucascode = re.findall('Course - (.*)',ucascode)[0]
#print(ucascode)
except:
ucascode = ''
#print(ucascode)
try:
tuition_fee = '16500'
# tuition_fee = remove_tags(tuition_fee)
# tuition_fee = tuition_fee.replace('£','')
# tuition_fee = tuition_fee.replace(',','')
# tuition_fee = tuition_fee.replace('*','')
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\r\n','')
# tuition_fee = tuition_fee.replace('\n','')
#
# tuition_fee = re.findall('(\d\d\d\d\d)',tuition_fee)[0]
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#print(tuition_fee)
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 2
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 1
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
item["alevel"] = alevel
item["ib"] = ib
item["ucascode"] = ucascode
#item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
#item["apply_pre"] = ''
yield item
def parse(self, response):
pass
# print(response.url)
item = UcasItem()
university = 'Bishop Grosseteste University'
try:
location = 'Dundee'
#location = remove_tags(location)
#print(location)
except:
location = 'N/A'
#print(location)
try:
department = response.xpath(
'//*[@id="bgu-single"]/div/article/div/div[1]/div[2]/div/table/tbody/tr[2]/td[2]'
).extract()[0]
department = remove_tags(department)
department = department.replace('\n\n', '\n')
department = department.replace('\r\n', '')
department = department.replace(' ', '')
department = department.replace(' ', '')
department = department.replace('\n', '')
#department = department.replace('Our Staff', '')
#print(department)
except:
department = ''
#print(department)
try:
degree_name = response.xpath(
'//*[@id="main-content"]/div[1]/div/div/div[2]/h1').extract(
)[0]
degree_name = remove_tags(degree_name)
#degree_name = degree_name.split()[-1]
degree_name = re.findall('\((.*)\).*', degree_name)[0]
#degree_name = re.findall('(.*) .*',degree_name)[0]
#degree_name = re.findall('\((.*)\)',degree_name)[0]
#degree_name = degree_name.replace('\n',degree_name)
#degree_name = degree_name.replace(' ','')
#print(degree_name)
except:
degree_name = 'N/A'
#print(degree_name)
try:
degree_overview_en = ''
degree_overview_en = remove_tags(degree_overview_en)
degree_overview_en = "<div><p>" + degree_overview_en + "</p></div>"
#print(degree_overview_en)
except:
degree_overview_en = ''
try:
programme_en = response.xpath(
'//*[@id="main-content"]/div[1]/div/div/div[2]/h1').extract(
)[0]
programme_en = remove_tags(programme_en)
#programme_en = programme_en.split()[1]
#programme_en = re.findall(' (.*)',programme_en)[0]
#programme_en = programme_en.replace(degree_name,'')
#programme_en = programme_en.replace(' ','')
#programme_en = programme_en.replace('\n', '')
#programme_en = re.findall((' '),'')[0]
#programme_en = re.findall("\(.*\)(.*)",programme_en)[0]
#programme_en = programme_en.replace('\n','')
#programme_en = programme_en.replace(' ','')
#print(programme_en)
except:
programme_en = 'N/A'
#print(programme_en)
try:
overview_en = response.xpath(
'//*[@id="bgu-single"]/div/article/div/div[1]/div[1]/div/p'
).extract()[0]
overview_en = remove_tags(overview_en)
#overview_en = overview_en.replace(' ','')
#overview_en = overview_en.replace('\n\n','\n')
overview_en = overview_en.replace('\n\n', '')
overview_en = overview_en.replace('\r\n', '')
overview_en = overview_en.replace('\n', '')
overview_en = '<div>' + overview_en + '</div>'
#overview_en = remove_tags(overview_en)
#print(overview_en)
except:
overview_en = 'N/A'
#print(overview_en)
try:
start_date = '9,10'
#print(start_date)
except:
start_date = ''
try:
modules_en = response.xpath(
'///*[@id="bgu-single"]/div/article/div/div[4]/div/div'
).extract()[0]
modules_en = remove_tags(modules_en)
modules_en = modules_en.replace('\n\n', '\n')
modules_en = modules_en.replace('\r\n', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace(' ', '')
modules_en = modules_en.replace('\n', '')
modules_en = "<div><p>" + modules_en + "</p></div>"
#print(modules_en)
except:
modules_en = 'N/A'
#print(modules_en)
try:
degree_requirements = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[2]/div[2]/div[1]/div[2]'
).extract()[0]
degree_requirements = remove_tags(degree_requirements)
degree_requirements = degree_requirements.replace(' ', '')
#print(degree_requirements)
except:
degree_requirements = ''
#print(degree_requirements)
try:
rntry_requirements_en = response.xpath(
'//*[@id="bgu-single"]/div/article/div/div[3]/div/div'
).extract()[0]
rntry_requirements_en = remove_tags(rntry_requirements_en)
rntry_requirements_en = "<div>" + rntry_requirements_en + "</div>"
rntry_requirements_en = rntry_requirements_en.replace('\n\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\r\n', '')
rntry_requirements_en = rntry_requirements_en.replace('\n', '')
rntry_requirements_en = rntry_requirements_en.replace(' ', '')
#rntry_requirements_en =rntry_requirements_en.replace(' ','')
#print(rntry_requirements_en)
except:
rntry_requirements_en = 'N/A'
#print(rntry_requirements_en)
try:
professional_background = response.xpath('').extract()
professional_background = remove_tags(professional_background)
except:
professional_background = ''
try:
require_chinese_en = ''
except:
require_chinese_en = ''
try:
ielts_desc = response.xpath(
'//*[@id="entry-requirements"]/div/section[2]/div/ul/li'
).extract()[0]
ielts_desc = remove_tags(ielts_desc)
#print(ielts_desc)
except:
ielts_desc = 'N/A'
#print(ielts_desc)
try:
#ielts = '6.5'
#ielts =remove_tags(ielts)
#ielts = re.findall('IELTS(.*)',ielts)[0]
ielts = '6.5'
#print(ielts)
except:
ielts = 0
#print(ielts)
try:
#ielts_l = '5.5'
ielts_l = '6.0'
#print(ielts_l)
#ielts_l = remove_tags(ielts_l)
except:
ielts_l = 0
try:
ielts_s = ielts_l
except:
ielts_s = ielts_l
try:
ielts_r = ielts_l
except:
ielts_r = ielts_l
try:
ielts_w = ielts_l
except:
ielts_w = ielts_l
try:
toefl_code = response.xpath('').extract()
toefl_code = remove_tags(toefl_code)
except:
toefl_code = 0
try:
toefl_desc = response.xpath('').extract()
toefl_desc = remove_tags(toefl_desc)
except:
toefl_desc = 0
try:
toefl = response.xpath('').extract()
toefl = remove_tags(toefl)
except:
toefl = 0
try:
toefl_l = response.xpath('').extrcat()
toefl_l = remove_tags(toefl_l)
except:
toefl_l = 0
try:
toefl_s = response.xpath('').extract()
toefl_s = remove_tags(toefl_s)
except:
toefl_s = 0
try:
toefl_r = response.xpath('').extract()
toefl_r = remove_tags(toefl_r)
except:
toefl_r = 0
try:
toefl_w = response.xpath('').extract()
toefl_w = remove_tags(toefl_w)
except:
toefl_w = 0
try:
interview_desc_en = response.xpath(
'//*[@id="entry-requirements-accordion-0"]/div[1]').extract(
)[0]
interview_desc_en = remove_tags(interview_desc_en)
interview_desc_en = interview_desc_en.replace('\n\n', '\n')
interview_desc_en = interview_desc_en.replace('\r\n', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace(' ', '')
interview_desc_en = interview_desc_en.replace('\n', '')
interview_desc_en = "<div>" + interview_desc_en + "</div>"
#print(interview_desc_en)
except:
interview_desc_en = 'N/A'
#print(interview_desc_en)
try:
work_experience_desc_en = response.xpath('').extract()
work_experience_desc_en = remove_tags(work_experience_desc_en)
except:
work_experience_desc_en = ''
try:
portfolio_desc_en = response.xpath('').extract()
portfolio_desc_en = remove_tags(portfolio_desc_en)
except:
portfolio_desc_en = ''
try:
career_en = response.xpath(
'//*[@id="bgu-single"]/div/article/div/div[8]/div').extract(
)[0]
career_en = remove_tags(career_en)
career_en = career_en.replace('\r\n', '')
career_en = career_en.replace(' ', '')
career_en = career_en.replace('\n', '')
career_en = "<div><span>" + career_en + "</span></div>"
#print(career_en)
except:
career_en = ''
#print(career_en)
try:
apply_desc_en = '<div>Starting your university application can be a complicated process, but don\'t worry - we’re here to help every step of the way. The easiest way to apply is to search below and select your course, then simply click the \'APPLY NOW\' button on the course page.</div>'
#apply_desc_en = remove_tags(apply_desc_en)
#apply_desc_en = "<div>" + apply_desc_en + "</div>"
#print(apply_desc_en)
except:
apply_desc_en = ''
try:
apply_documents_en = ''
#apply_documents_en = remove_tags(apply_documents_en)
except:
apply_documents_en = ''
apply_fee = 12
#other = ''
try:
apply_proces_en = response.xpath('').extract()
except:
apply_proces_en = ''
try:
duration = 1
#duration = remove_tags(duration)
#duration = remove_tags(duration)
#duration = re.findall('(\d) Years',duration)[0]
# if '36' in duration:
# duration = '3'
# elif '16' in duration:
# duration = '1'
# elif '12' in duration:
# duration = '1'
# elif '3' in duration:
# duration = '3'
# elif '2' in duration:
# duration = '2'
# elif '1' in duration:
# duration = '1'
# elif 'two' in duration:
# duration = '2'
# else:
# duration = '1'
# #print(duration)
except:
duration = 0
#print(duration)
try:
other = response.xpath(
'//*[@id="what-you-will-study"]/div/div[1]/div[1]/div[2]/div/div[2]/div/div/a'
).extract()[0]
other = remove_tags(other)
#print('成功'+ other + response.url)
except:
other = ''
#print('失败' + other)
try:
ib = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div[1]/table[1]/tbody/tr[11]/td[2]'
).extract()[0]
ib = remove_tags(ib)
#print(ib)
except:
ib = ''
#print(ib)
try:
alevel = response.xpath(
'//*[@id="tab-Entry_Requirements"]/div/div[1]/div/table[1]'
).extract()[0]
alevel = remove_tags(alevel)
alevel = re.findall("(\w\w\w) at A Level", alevel)[0]
#print(alevel)
except:
alevel = 'N/A'
#print(alevel)
try:
ucascode = response.xpath(
'/html/body/div[3]/div[1]/div/div/div[2]/div/div[1]/div[1]/div[2]'
).extract()[0]
ucascode = remove_tags(ucascode)
#print(ucascode)
except:
ucascode = ''
#print(ucascode)
try:
tuition_fee = '12500'
# tuition_fee = tuition_fee.replace(' ','')
# tuition_fee = tuition_fee.replace('\n','')
# tuition_fee = re.findall('Full-time international students: £(.*) paStudents',tuition_fee)[0]
# tuition_fee = int(tuition_fee)
#print(tuition_fee)
except:
tuition_fee = 0
#print(tuition_fee)
try:
teach_time = response.xpath(
'//*[@id="bgu-single"]/div/article/div/div[1]/div[2]/div'
).extract()[0]
teach_time = remove_tags(teach_time)
if 'full' in teach_time:
teach_time = 'fulltime'
elif 'Full' in teach_time:
teach_time = 'fulltime'
else:
teach_time = 'parttime'
#print(teach_time)
except:
teach_time = 'N/A'
#print(teach_time)
teach_type = 'taught'
try:
assessment_en = response.xpath(
'//div[@class = "cc-assessment mrt_30 bg-primary text-white p_20"]'
).extract()[0]
assessment_en = remove_tags(assessment_en)
#assessment_en = assessment_en.replace('\n','')
assessment_en = assessment_en.replace('\r\n', '')
assessment_en = assessment_en.replace(' ', '')
assessment_en = "<div>" + assessment_en + "</div>"
#print(assessment_en)
except:
assessment_en = 'N/A'
#print(assessment_en)
require_chinese_en = '<div>UG A recognised International Foundation Year from a UK institution or a Chinese institution when following a validated UK syllabus. OR Successfully completed first year of a Chinese University degree OR 2 or 3 year Diploma (Zhuanke or Da Zhuan) with a minimum final grade of 70% or equivalent PG Completion of a Bachelor degree from an accredited Chinese university with 75% or higher (GPA 2.9 or above) If you find that your qualifications do not meet our entry requirements, relevant experience and completion of one of BGU’s online pre-arrival courses can also be taken into account to meet the entry criteria.</div>'
item["university"] = university
item["location"] = location
item["department"] = department
item["degree_type"] = 2
item["degree_name"] = degree_name
#item["degree_overview_en"] = degree_overview_en
item["programme_en"] = programme_en
item["overview_en"] = overview_en
item["teach_time"] = 1
item["start_date"] = start_date
item["modules_en"] = modules_en
item["career_en"] = career_en
item["application_open_date"] = '9'
item["deadline"] = ''
item["apply_pre"] = '£'
item["apply_fee"] = apply_fee
#item["rntry_requirements_en"] = rntry_requirements_en
item["degree_requirements"] = degree_requirements
item["tuition_fee_pre"] = '£'
#item["major_requirements"] = rntry_requirements_en
item["professional_background"] = professional_background
item["ielts_desc"] = ielts_desc
item["ielts"] = ielts
item["ielts_l"] = ielts_l
item["ielts_s"] = ielts_l
item["ielts_r"] = ielts_l
item["ielts_w"] = ielts_l
item["toefl_code"] = toefl_code
item["toefl_desc"] = toefl_desc
item["toefl"] = toefl
item["toefl_l"] = toefl_l
item["toefl_s"] = toefl_s
item["toefl_r"] = toefl_r
item["toefl_w"] = toefl_w
item["work_experience_desc_en"] = work_experience_desc_en
item["interview_desc_en"] = interview_desc_en
item["portfolio_desc_en"] = portfolio_desc_en
item["apply_desc_en"] = apply_desc_en
item["apply_documents_en"] = apply_documents_en
item["other"] = other
item["url"] = response.url
item["gatherer"] = 'weihongbo'
item["apply_proces_en"] = apply_proces_en
item["batch_number"] = 3
item["finishing"] = 0
stime = time.time()
create_time = time.strftime("%Y-%m-%dT%H:%M:%S",
time.localtime(float(stime)))
#print(create_time)
item["create_time"] = create_time
item["import_status"] = 0
item["duration"] = duration
item["tuition_fee"] = tuition_fee
item["update_time"] = create_time
#item["alevel"] = alevel
#item["ib"] = ib
#item["ucascode"] = ucascode
item["rntry_requirements"] = rntry_requirements_en
item["require_chinese_en"] = require_chinese_en
item["teach_time"] = teach_time
item["teach_type"] = teach_type
item["assessment_en"] = assessment_en
#item["apply_pre"] = ''
yield item
