def import_companies(self):
csv_file = self.read_file('importer/data/companies.csv')
print 'Importing Companies...'
for row in csv_file:
try:
Company.objects.get(name=row[0])
print 'Company', row[0], 'already registered.'
except ObjectDoesNotExist:
company = Company()
company.name = row[0]
company.save()
def post(self):
name = self.get_argument('name')
code = self.get_argument('code')
address = self.get_argument('address')
parent = self.get_argument('parent')
company = self.session.query(Company).filter(
Company.code == code).first()
if company:
data = time.strftime("%Y-%m-%d")
company = Company(
id=company.id,
name=name,
code=code,
address=address,
parent=parent,
data=data,
)
self.session.merge(company)
self.session.commit()
self.redirect('/fra/com')
else:
err = '公司编码不存在'
self.render("fra_com_add.html",
auth_user=self.current_user,
err=err)
def get_subsidiary(self, lot):
subsidiary = Company()
lot = [snippet.text for snippet in lot]
cmps = filter(lambda snippet : self.is_company(snippet), lot)
for t in lot:
s = self.get_state(t)
if s:
subsidiary.location = first_letter_caps(s)
if not subsidiary.location:
for t in lot:
c = self.get_country(t)
if c:
subsidiary.location = first_letter_caps(c)
if len(cmps) > 0:
subsidiary.name = clean_name(cmps[0])
return subsidiary
def get_10k_docs(self, urls):
"""
Steps:
Get the 10K documents for a list of URLs in the index file
Find the company name and addresses in the 10k filing
Look for an exhibit 21 document within the 10k
If the exhibit 21 exists, get that, then pull the subsidiaries out and add them
to the newly created company
"""
companies = []
for url in urls:
company = Company()
full = BASE_URL + '/Archives/%s-index.htm' % url[0:len(url)-4]
company.url = full
contents = self.downloader.get_url(full)
try:
soup = BeautifulSoup(contents, convertEntities=BeautifulSoup.HTML_ENTITIES)
company_name = soup.findAll('span', attrs={'class' : 'companyName'})
company_name = re.search('(?<=>).*?(?=<)', str(company_name[0]), re.DOTALL).group(0)
except IndexError:
self.downloader.purge(full)
contents = self.downloader.get_url(full)
soup = BeautifulSoup(contents, convertEntities=BeautifulSoup.HTML_ENTITIES)
company_name = soup.findAll('span', attrs={'class' : 'companyName'})
company_name = re.search('(?<=>).*?(?=<)', str(company_name[0]), re.DOTALL).group(0)
company.name = clean_name(company_name)
existing = self.backend.get_company(company.name)
if not existing:
addresses = soup.findAll('div', attrs={'class' : 'mailer'})
for address in addresses:
if "business address" in address.text.lower():
#This is the business address
company.business_address = clean_addr(self.extract_address(address))
pass
if "mailing address" in address.text.lower():
company.mailing_address = clean_addr(self.extract_address(address))
table_rows = soup.findAll('tr')
for row in table_rows:
#For each row in the table rows, see if it is likely a row that contains
#the exhibit 21.1 (subsidiares) form. If it is, grab the URL and load the page
#
if re.search(EXHIBIT21_WORDS, row.text, re.IGNORECASE):
ex21_url = row.find('a').get('href')
try:
company.subsidiaries = self.get_exhibit21(BASE_URL + ex21_url)
company.exhibit21_url = BASE_URL + ex21_url
except TypeError:
#Sometimes beautifulsoup trys to concatenate a str and None?
pass
self.backend.add(company)
import json
from models.models import Company
if __name__ == '__main__':
with open('sample_companies.json') as f:
sample_data_as_json = json.load(f)
for element in sample_data_as_json:
company = Company(name=element['name'],
strasse=element['strasse'],
plz=element['plz'],
ort=element['ort'],
land=element['land'])
company.save()
def insert():
print('function is started')
i = 0
jobs = vacancies()
months = [
'Yanvar', 'Fevral', 'Mart', 'Aprel', 'May', 'İyun', 'İyul', 'Avqust',
'Sentyabr', 'Oktyabr', 'Noyabr', 'Dekabr'
]
new_users = {}
new_positions = {}
for job in jobs:
replacejob = Jobs.query.filter_by(boss_id=job['vac_id']).first()
if replacejob:
continue
else:
#--------------------Position Category and Subcategory---------------
position = Positions.query.filter_by(name=job['title']).first()
if position:
position_id = position.id
category_id = position.category_id
subcategory_id = position.subcategory_id
else:
if job['title'] in new_positions.values():
values = list(new_positions.values())
keys = list(new_positions.keys())
ids = keys[values.index(job['title'])]
ids = ids.split('-')
position_id = int(ids[0])
subcategory_id = int(ids[2])
category_id = int(ids[1])
else:
cat_other = Categories.query.filter_by(
name='Müxtəlif').first()
if cat_other:
category_id = cat_other.id
else:
category = Categories(
name='Digər',
is_active=1,
sort_order=Categories.query.count() + 1,
slug='miscellanious',
is_home=1)
db.session.add(category)
db.session.commit()
category = Categories.query.order_by(
Categories.id.desc()).first()
category_id = category.id
sub_other = Subcategories.query.filter_by(
name='Digər').first()
if sub_other:
subcategory_id = sub_other.id
else:
sub = Subcategories(
name='Digər',
is_active=1,
sort_order=Categories.query.count() + 1,
slug='miscellanious',
category_id=category_id)
db.session.add(sub)
db.session.commit()
sub = Subcategories.query.order_by(
Subcategories.id.desc()).first()
subcategory_id = sub.id
position = Positions(name=job['title'],
is_active=1,
sort_order=Positions.query.count() +
1,
category_id=category_id,
subcategory_id=subcategory_id)
db.session.add(position)
db.session.flush()
position_id = position.id
db.session.commit()
new_positions.update({
str(position_id) + '-' + str(category_id) + '-' + str(subcategory_id):
job['title']
})
#---------------------Company----------------
mail = job['email'].split('@')
search = "%{}%".format(mail[1])
user = User.query.filter_by(email=job['email'],
is_company=1).first()
if user:
company = Company.query.filter_by(user_id=user.id).first()
company_id = company.id
user_id = user.id
else:
if job['email'] in new_users.values():
values = list(new_users.values())
keys = list(new_users.keys())
ids = keys[values.index(job['email'])] # 'foo'
ids = ids.split('-')
user_id = int(ids[0])
company_id = int(ids[1])
else:
user = User(name=job['email'],
email=job['email'],
is_company=1)
db.session.add(user)
db.session.flush()
user_id = user.id
company = Company(name=job['company'], user_id=user.id)
db.session.add(company)
db.session.flush()
company_id = company.id
db.session.commit()
new_users.update(
{str(user_id) + '-' + str(company_id): job['email']})
# --------------------Deadline---------------
time = job['expires_on'].split(' ')
time[1] = time[1].strip(',')
month = months.index(time[0]) + 1
deadline = time[2] + '-' + str(month) + '-' + time[1]
#---------------------SALARY-----------------
salary = job['salary']
salary = salary.strip(' AZN')
salary_list = salary.split('-')
if len(salary_list) > 1:
print(salary)
print(salary_list)
if salary_list[0] == '':
salary = 3
minsalary = None
salaryfix = None
maxsalary = None
else:
salary = 2
minsalary = salary_list[0]
print('minsalary=' + minsalary)
maxsalary = salary_list[1]
salaryfix = None
else:
if salary_list[0] == '-':
salary = 3
minsalary = None
salaryfix = None
maxsalary = None
else:
salary = 1
salaryfix = salary_list[0]
minsalary = None
maxsalary = None
# -------------------Age-----------------
if 'minimum' in job['age']:
age = job['age'].split(' ')
agemin = Select.query.filter_by(title=age[1],
type_id=11).first()
if agemin:
agemin_id = agemin.id
else:
addAge = Select(type=11, title=age[0])
db.session.add(addAge)
db.session.flush()
agemin_id = addAge.id
agemax = Select.query.filter_by(title='Digər',
type_id=11).first()
if agemax:
agemax_id = agemax.id
else:
addAge = Select(type_id=11, title='Digər')
db.session.add(addAge)
db.session.flush()
agemax_id = addAge.id
elif 'maks' in job['age']:
age = job['age'].split(' ')
agemax = Select.query.filter_by(title=age[1],
type_id=11).first()
if agemax:
agemax_id = agemax.id
else:
addAge = Select(type=11, title=age[0])
db.session.add(addAge)
db.session.flush()
agemax_id = addAge.id
agemin = Select.query.filter_by(title='Digər',
type_id=11).first()
if agemin:
agemin_id = agemin.id
else:
addAge = Select(type_id=11, title='Digər')
db.session.add(addAge)
db.session.flush()
agemin_id = addAge.id
else:
age = job['age'].split('-')
age[1] = age[1].strip(' yaş')
agemin = Select.query.filter_by(title=age[0],
type_id=11).first()
if agemin:
agemin_id = agemin.id
else:
addAge = Select(type=11, title=age[0])
db.session.add(addAge)
db.session.flush()
agemin_id = addAge.id
agemax = Select.query.filter_by(title=age[1],
type_id=11).first()
if agemax:
agemax_id = agemax.id
else:
addAge = Select(type_id=11, title=age[1])
db.session.add(addAge)
db.session.flush()
agemax_id = addAge.id
# EDUCATION ID
education = Select.query.filter_by(title=job['education'],
type_id=10).first()
if education:
edu_level = education.id
else:
education = Select.query.filter_by(title='Digər',
type_id=10).first()
if education:
edu_level = education.id
else:
education = Select(
title='Digər',
type_id=10,
)
db.session.add(education)
db.session.flush()
edu_level = education.id
db.session.commit()
#----------------------------------EXPERIENCE ID--------------------------
experience_list = job['experience'].split(' ')
if 'aşağı' in job['experience']:
experience = Select.query.filter_by(title='Vacib deyil',
type_id=17).first()
experience_id = experience.id
elif 'artıq' in job['experience']:
experience = Select.query.filter_by(title='5-10 il',
type_id=17).first()
experience_id = experience.id
else:
experience_format = experience_list[0] + '-' + experience_list[
2] + ' il'
experience = Select.query.filter_by(title=experience_format,
type_id=17).first()
experience_id = experience.id
#-----------------------------------------POSITION LEVEL----------------------------------------------------------------
position_level = Select.query.filter_by(title='Digər',
type_id=13).first()
if position_level:
position_level_id = position_level.id
else:
position_level = Select(title='Digər', type_id=13)
db.session.add(position_level)
db.session.flush()
position_level_id = position_level.id
db.session.commit()
#-----------------------------------------WORKING TYPE------------------------------------------------------------------
working = Select.query.filter_by(title='Digər', type_id=14).first()
if working:
workingtype_id = working.id
else:
working = Select(title='Digər', type_id=14)
db.session.add(working)
db.session.flush()
workingtype_id = working.id
db.session.commit()
# FIND LOCATION
location = Select.query.filter_by(title=job['region'],
type_id=15).first()
if location:
location_id = location.id
else:
location = Select(title=job['region'], type_id=15)
db.session.add(location)
db.session.flush()
location_id = location.id
db.session.commit()
#-----------------------------------------JOB INFORMATION--------------------------------------------------------------
job['information'] = job['information'] + "\n- Əlaqə üçün: " + job[
'email']
job['information'] = job['information'].replace("\n", "<br>")
job['information'] = job['information'].replace("[email", '')
job['information'] = job['information'].replace("([email", '')
job['information'] = job['information'].replace(
"protected]", job['email'])
job['information'] = job['information'].replace(
"protected])", job['email'])
job['requirements'] = job['requirements'].replace('-', '<br> -')
job['requirements'] = job['requirements'].replace("([email", '')
job['requirements'] = job['requirements'].replace("[email", '')
job['requirements'] = job['requirements'].replace(
"protected])", job['email'])
job['requirements'] = job['requirements'].replace(
"protected]", job['email'])
job = Jobs(title=job['title'],
aboutjob=job['information'],
qualifications=job['requirements'],
educationlevel_id=edu_level,
boss_id=job['vac_id'],
jobexperience_id=experience_id,
category_id=category_id,
subcategory_id=subcategory_id,
position_id=position_id,
lang_id=2,
location_id=location_id,
positionlevel_id=position_level_id,
workingtype_id=workingtype_id,
jobtype=1,
status=2,
is_active=1,
employeecount=1,
deadline=deadline,
agemin_id=agemin_id,
agemax_id=agemax_id,
salary=salary,
salaryfix=salaryfix,
maxsalary=maxsalary,
minsalary=minsalary,
company_id=company_id,
user_id=user_id,
unicid=id_generator(31))
db.session.add(job)
db.session.commit()
session.clear()
db.session.remove()
i = i + 1
print(i)
print('finish')
return 'okay', 200