def parse_companyjobs(source_company_id, item, sourceId):
source_jobs = []
logger.info("source_company_id is %s", source_company_id)
d = pq((html.fromstring(item["content"].decode("utf-8"))))
# logger.info("this page has %s jobs", position_type, len(jobs))
for li in d('.job-info'):
dj = pq(li)
job_link = dj('a.title').attr("href")
job_key = job_link.split("/")[-1].replace(".shtml", "")
born_time = dj('time').attr('title')
position = dj('a.title').attr("title")
city = dj('.condition span:nth-child(2)').text().split('-')[0]
work_year = dj('.condition span:nth-child(4)').text()
education = dj('.condition span:nth-child(3)').text()
logger.info("%s - %s - %s", city, work_year, education)
salary = dj('.condition span:nth-child(1)').text()
# update_time = born_time
domain = 0
location_id = 0
location_new = parser_db_util.get_location(city)
if location_new != None:
location_id = location_new["locationId"]
education_type = 0
if '大专' in education:
education_type = 6020
elif '本科' in education:
education_type = 6030
elif '硕士' in education:
education_type = 6040
elif '博士' in education:
education_type = 6050
workYear_type = 7000
if '应届' in work_year:
workYear_type = 7010
elif '1年以下' in work_year:
workYear_type = 7020
elif '1年以上' in work_year or '2年以上' in work_year:
workYear_type = 7030
elif '3年以上' in work_year:
workYear_type = 7040
elif '5年以上' in work_year or '6年以上' in work_year or '8年以上' in work_year:
workYear_type = 7050
elif '10年以上' in work_year:
workYear_type = 7060
logger.info("born time: %s", born_time)
try:
born_time = born_time.replace("发布于", "")
if born_time.find("月") >= 0:
logger.info("neeewdate:%s", born_time)
update_time = datetime.datetime.strptime(
born_time, "%Y年%m月%d日")
else:
update_time = datetime.datetime.now()
except Exception, e:
logger.info(e)
update_time = datetime.datetime.now()
source_job = {
"source": SOURCE,
"sourceId": job_key,
"recruit_company_id": str(source_company_id),
"position": position,
"salary": salary,
"description": None,
"domain": domain,
"locationId": location_id,
"educationType": education_type,
"workYearType": workYear_type,
"startDate": update_time,
"updateDate": update_time,
"jobNature": None,
"positionAdvantage": None,
"companyLabelList": None,
"financeStage": None,
"district": None,
}
logger.info("job_key: %s", job_key)
for i in source_job:
logger.info("%s->%s", i, source_job[i])
source_jobs.append(source_job)
def parse_investor(item):
logger.info("parse_investor")
investor_key = item["key"]
c = item["content"]["basic"]["data"]
establish_date = None
if c.has_key("startDate"):
d = time.localtime(c["startDate"]/1000)
if d.tm_year > 1980:
establish_date = datetime.datetime(d.tm_year,d.tm_mon,d.tm_mday)
addresses = []
for ad in c.get("addresses",[]):
address1 = None
address2 = None
if ad.has_key("city"):
address1 = ad["city"]
if ad.has_key("address"):
address2 = ad["address"]
location_id = 0
if address1 is not None and address1.strip()!="":
city = address1
if city != None:
location = parser_db_util.get_location(formCityName(city))
if location != None:
location_id= location["locationId"]
if location_id==0 and address2 != None and address2.strip()!="":
city = address2
if city != None:
location = parser_db_util.get_location(formCityName(city))
if location != None:
location_id = location["locationId"]
if (address2 is not None and address2.strip()!="") or (ad.get("phone",None) is not None and ad.get("phone",None).strip() !="") or \
ad.get("email", None) is not None and ad.get("email", None).strip() != "" :
addresses.append({
"locationId": location_id,
"address": address2,
"phone": ad.get("phone",None) if ad.get("phone",None) is not None and ad.get("phone",None).strip() !="" else None,
"email": ad.get("email", None) if ad.get("email", None) is not None and ad.get("email",
None).strip() != "" else None,
})
name = c["nameAbbr"]
fullName = c["name"]
if (name is None or name.strip() == "") and (fullName is None or fullName.strip() ==""):
logger.info("*******************wrong thing")
return {
"wrong": 1,
"key": investor_key
}
else:
return {
"name": fullName if name is None or name.strip() == "" else name,
"fullName": None if fullName is None or fullName.strip() == "" else fullName,
"description": None if c.get("intro",None) is None or c.get("intro",None).strip() == "" else c["intro"],
"website": None if c.get("website",None) is None or c.get("website",None).strip() == "" else c["website"],
"logo": None if c.get("logo",None) is None or c.get("logo",None).strip() == "" else c["logo"],
"source": SOURCE,
"sourceId": str(investor_key),
"wechatId": None if c.get("weixin",None) is None or c.get("weixin",None).strip() == "" else c["weixin"],
"weibo": None if c.get("weibo",None) is None or c.get("weibo",None).strip() == "" else c["weibo"],
"enName": None if c.get("enNameAbbr",None) is None or c.get("enNameAbbr",None).strip() == "" else c["enNameAbbr"],
"enFullName": None if c.get("enName",None) is None or c.get("enName",None).strip() == "" else c["enName"],
"establishDate": establish_date,
"addresses": addresses
}
def parse_company(item):
logger.info("parse_company")
company_key = item["postdata"]["id"]
#company basic info
c = item["data"]["basic"]
tags = c["tags"]
tags_str = tags.replace("|",",")
logo=c["icon"]
if logo.find("product_default.png") >= 0:
logo = None
establish_date = None
if c.has_key("open_time"):
try:
establish_date = datetime.datetime.strptime(c["open_time"], "%Y-%m-%d")
except:
pass
address1 = None
address2 = None
if c.has_key("city"):
address2 = c["city"]
if c.has_key("province"):
address1 = c["province"]
location_id = 0
if address2!=None and address2.strip()!="":
location = parser_db_util.get_location(address2)
if location != None:
location_id= location["locationId"]
if location_id==0 and address1 != None and address1.strip()!="":
location = parser_db_util.get_location(address1)
if location != None:
location_id = location["locationId"]
fullName = c["company"]
if fullName is None or fullName.strip() == "":
fullName = None
else:
fullName = fullName.replace("_","")
idx = fullName.rfind(u"公司")
if idx != -1:
fullName = fullName[:(idx+len(u"公司"))]
fullName = name_helper.company_name_normalize(fullName)
name = c["product"]
desc = ""
brief = ""
productDesc = None
modelDesc = None
operationDesc = None
teamDesc = None
marketDesc = None
compititorDesc = None
advantageDesc = None
planDesc = None
otherDesc = None
if c.has_key("desc"): # 其他
# otherDesc = c["intro"].strip()
desc = c["desc"].strip()
if c.has_key("yewu"): # 其他
# otherDesc = c["intro"].strip()
brief = c["yewu"].strip()
if name is None or fullName is None:
return {
"status": "No_Name",
}
artifacts = []
websites = []
if c.has_key("gw_link") is True and c["gw_link"].strip() !="" and c["gw_link"] not in websites:
websites.append(c["gw_link"])
if c.has_key("source_gw_link") is True and c["source_gw_link"].strip() != "" and c["source_gw_link"] not in websites:
websites.append(c["source_gw_link"])
if item["data"].has_key("productinfos") is True:
for pi in item["data"]["productinfos"]:
if pi.has_key("link") is True and pi["link"].strip() !="" and pi["link"] not in websites:
websites.append(pi["link"])
for website in websites:
type, app_market, app_id = url_helper.get_market(website)
if type == 4010:
if item["url"] != website and website.find("qimingpian.com") == -1:
flag, domain = url_helper.get_domain(website)
if flag is not None:
if flag is False:
domain = None
artifacts.append({
"type": 4010,
"name": name,
"description": brief,
"link": website,
"domain": domain
})
elif type == 4020 or type == 4030:
domain = None
if domain is not None:
artifacts.append({
"type": type,
"name": name,
"description": brief,
"link": website,
"domain": domain
})
elif type == 4040:
domain = app_id
if domain is not None:
artifacts.append({
"type": 4040,
"name": name,
"description": brief,
"link": website,
"domain": domain
})
elif type == 4050:
domain = None
if app_market == 16010 or app_market == 16020:
android_app = parser_db_util.find_android_market(app_market, app_id)
if android_app:
domain = android_app["apkname"]
else:
domain = app_id
if domain is not None:
artifacts.append({
"type": 4050,
"name": name,
"description": brief,
"link": website,
"domain": domain
})
return {
"name": name,
"fullName": fullName,
"description": desc,
"productDesc": productDesc,
"modelDesc": modelDesc,
"operationDesc": operationDesc,
"teamDesc": teamDesc,
"marketDesc": marketDesc,
"compititorDesc": compititorDesc,
"advantageDesc": advantageDesc,
"planDesc": planDesc,
"otherDesc": otherDesc,
"brief": brief,
"round": 0,
"roundDesc": None,
"companyStatus": 2010,
'fundingType': 0,
"locationId": location_id,
"address": None,
"phone": None,
"establishDate": establish_date,
"logo": logo,
"source": SOURCE,
"sourceId": company_key,
"field": None,
"subField": None,
"tags": tags_str,
"headCountMin": None,
"headCountMax": None,
"artifacts": artifacts,
}
def parse_company(item):
logger.info("parse_neeq_stock")
# logger.info(item)
company_key = item["sourceId"]
#company basic info
c = item["baseinfo"]
field = c.get("industry",None)
result = parser_db_util.get_location(c["area"].replace("市",""))
if result != None:
location_id = result["locationId"]
else:
location_id = None
fullName = c["name"]
logger.info("parsing :%s|%s", fullName, company_key)
desc = None
brief = None
try:
desc = item["memectBrief"][u"公司简介"]
except:
pass
if desc is None:
try:
desc = item["jqkaBrief"]["desc"]
except:
pass
try:
brief = item["memectBrief"][u"主营产品"].split("。")[0]
logger.info(len(brief))
if len(brief) > 99:
brief = None
except:
pass
if brief is None:
try:
brief = item["jqkaBrief"]["brief"].split("。")[0]
logger.info(len(brief))
if len(brief) > 99:
brief = None
if brief is not None and brief.strip() == "-":
brief = None
except:
pass
productDesc = None
modelDesc = None
operationDesc = None
teamDesc = None
marketDesc = None
compititorDesc = None
advantageDesc = None
planDesc = None
otherDesc = None
return {
"name": c["shortname"].replace('0',"").replace('7',"").replace('8',"").replace('9',"").replace('1',"").replace('2',"").replace('3',"").replace('4',"").replace('5',"").replace('6',""),
"fullName": fullName.strip(),
"description": desc,
"productDesc": productDesc,
"modelDesc": modelDesc,
"operationDesc": operationDesc,
"teamDesc": teamDesc,
"marketDesc": marketDesc,
"compititorDesc": compititorDesc,
"advantageDesc": advantageDesc,
"planDesc": planDesc,
"otherDesc": otherDesc,
"brief": brief,
"round": 0,
"roundDesc": None,
"companyStatus": 2010,
'fundingType': 0,
"locationId": location_id,
"address": c.get("address",None),
"phone": str(c["phone"]) if c.has_key("phone") and c["phone"] is not None and str(c["phone"]).strip()!="" else None,
"establishDate": None,
"logo": None,
"source": SOURCE,
"sourceId": company_key,
"field": field,
"subField": None,
"tags": None,
"headCountMin": None,
"headCountMax": None,
"englishName": c["englishName"] if c.has_key("englishName") else None
}
def parse_company(item):
# logger.info("parse_company")
d = pq(html.fromstring(item['content'].decode("utf-8")))
company_key = item["key"]
# company basic info
tags = []
for tag in d('.word_list').text().split():
if tag.strip() not in tags: tags.append(tag)
tags_str = ",".join(tags)
logo = d('.peoimg img').attr('src')
if logo:
logo = logo.replace("https://", "http://")
establish_date = None
time_content = d('.time_content li:last-child')
if d(time_content)('.upword').text().find('成立') > 0:
establish_date = d(time_content)('.time_up').text()
establish_date = datetime.datetime.strptime(establish_date, '%Y-%m-%d')
companyName = d('.company_div h5').text()
city = name_helper.get_location_from_company_name(companyName)[0]
location_id = 0
if city != None:
location = parser_db_util.get_location(city)
if location != None:
location_id = location["locationId"]
# logger.info("locationid =%s",location_id)
fullName = companyName.replace("_", "")
fullName = name_helper.company_name_normalize(fullName)
desc = d('#intro_srocll p').text()
productDesc = ''
website = ''
for p in d('.procont_lis p'):
if d(p).text().find('官网') > 0 and d(p)('a').attr('href') is not None:
website = d(p)('a').attr('href')
continue
productDesc += d(p).text() + '\n'
if desc == '' or desc is None: desc = productDesc
shortName = d('.peo_center h4').text().split('：')[0].split(':')[0].split(
'——')[0].split('，')[0].split('｜')[0]
companyResult = {}
# isCompany
# print companyName,company_key, ',', name_helper.name_check(companyName)[1], ',', len(website)>0
if name_helper.name_check(companyName)[1] == True:
# English name
if name_helper.name_check(shortName)[0] == False:
pass
else:
cnt = 0
for s in shortName:
if s in companyName: cnt += 1
if not cnt > 2:
shortName = companyName
else:
if not len(website) > 0:
return 0
else:
companyResult['fakeName'] = fullName
fullName = None
companyResult.update({
"name": shortName,
"fullName": fullName,
"description": desc,
"round": 0,
"roundDesc": None,
"companyStatus": 2010,
'fundingType': 0,
"locationId": location_id,
"address": None,
"phone": None,
"establishDate": establish_date,
"logo": logo,
"source": SOURCE,
"sourceId": company_key,
"field": None,
"subField": None,
"tags": tags_str,
"headCountMin": None,
"headCountMax": None,
"brief": None,
"website": website,
})
return companyResult
def parse_company(item):
name = item['name']
logger.info('parse_company:%s' % name)
c = item['content']['company_base']
company_key = item['key']
tags_str = None
tags = c['tags']
tagss = []
if tags:
for tag in tags:
tagss.append(tag['value'])
tags_str = ','.join(tagss)
# logger.info('tags：%s'%tags_str)
logo = c['logo']
if logo:
logo = "https://crunchbase-production-res.cloudinary.com/image/upload/c_lpad,h_100,w_100,f_auto,b_white,q_auto:eco/%s" % logo
# logger.info('logo：%s'%logo)
establish_date = None
if c['overview_fields'].has_key('founded_on'):
establish_date = c['overview_fields']['founded_on']['value']
if establish_date:
es = establish_date.split('-')
int_es = map(int, es)
if int_es[0] > 1980:
establish_date = datetime.datetime(int_es[0], int_es[1],
int_es[2])
else:
establish_date = None
# logger.info('establish_date：%s'%establish_date)
locaotions_str = None
locationss = []
if c['locations'].has_key('location_identifiers'):
locations = c['locations']['location_identifiers']
if locations:
for locat in locations:
locationss.append(locat['value'])
locaotions_str = ','.join(locationss)
# logger.info('locaotions_str：%s'%locaotions_str)
location_id = 421
contry = '国外'
if locaotions_str.lower().find('china') >= 0:
contry = '中国'
location = parser_db_util.get_location(contry)
if location != None:
location_id = location["locationId"]
# logger.info('location_id：%s'%location_id)
brief = None
if c['locations'].has_key('short_description'):
brief = c['locations']['short_description']
# logger.info('brief:%s'%brief)
desc = ""
descriptions = c['description']
if descriptions.has_key('description'):
desc = descriptions['description']
fullName = None
productDesc = None
modelDesc = None
operationDesc = None
teamDesc = None
marketDesc = None
compititorDesc = None
advantageDesc = None
planDesc = None
otherDesc = None
headCountMin = None
headCountMax = None
return {
"name":
name,
"fullName":
fullName if fullName is not None and fullName.strip() != "" else None,
"description":
desc,
"productDesc":
productDesc,
"modelDesc":
modelDesc,
"operationDesc":
operationDesc,
"teamDesc":
teamDesc,
"marketDesc":
marketDesc,
"compititorDesc":
compititorDesc,
"advantageDesc":
advantageDesc,
"planDesc":
planDesc,
"otherDesc":
otherDesc,
"brief":
brief,
"round":
0,
"roundDesc":
None,
"companyStatus":
2010,
'fundingType':
0,
"locationId":
location_id,
"address":
None,
"phone":
None,
"establishDate":
establish_date,
"logo":
logo,
"source":
SOURCE,
"sourceId":
company_key,
"field":
None,
"subField":
None,
"tags":
tags_str,
"headCountMin":
headCountMin,
"headCountMax":
headCountMax
}
def process(crawler_job, key, content, source_company_id):
# logger.info(content)
# if has_content(content):
if 1:
j = json.loads(content)
positions = j['data']['list']
domain = 0
jobs = []
for p in positions:
logger.info(p['url'])
key = p['url'].split('job/')[-1].split('.shtml')[0]
location_id = 0
location_new = parser_db_util.get_location(p['city'].split('-')[0])
if location_new != None:
location_id = location_new["locationId"]
education = p['eduLevel']
education_type = 0
if '大专' in education:
education_type = 6020
elif '本科' in education:
education_type = 6030
elif '硕士' in education:
education_type = 6040
elif '博士' in education:
education_type = 6050
work_year = p['workYear']
workYear_type = 7000
if '应届' in work_year:
workYear_type = 7010
elif '1年以下' in work_year:
workYear_type = 7020
elif '1年以上' in work_year or '2年以上' in work_year:
workYear_type = 7030
elif '3年以上' in work_year:
workYear_type = 7040
elif '5年以上' in work_year or '6年以上' in work_year or '8年以上' in work_year:
workYear_type = 7050
elif '10年以上' in work_year:
workYear_type = 7060
update_time = datetime.datetime.strptime(p['time'], "%Y年%m月%d日")
source_job = {
"source": SOURCE,
"sourceId": key,
"recruit_company_id": str(source_company_id),
"position": p['title'],
"salary": p['salary'],
"description": None,
"domain": domain,
"locationId": location_id,
"educationType": education_type,
"workYearType": workYear_type,
"startDate": update_time,
"updateDate": update_time,
"jobNature": None,
"positionAdvantage": None,
"companyLabelList": None,
"financeStage": None,
"district": None,
}
logger.info(json.dumps(source_job, ensure_ascii=False, cls=util.CJsonEncoder))
jobs.append(source_job)
logger.info(len(jobs))
if len(jobs) > 0:
save_job_mongo(jobs)
def parse_company(item):
logger.info("parse_company")
company_key = item["key"]
#company basic info
c = item["content"]["company_base"]["data"]["company"]
#check if page is under development or is completed(CREATED)
if c["status"] == "INIT":
return {
"status": c["status"],
}
tags = item["content"]["company_base"]["data"]["tags"]
tags2 = []
for tag in tags:
tags2.append(tag["name"])
tags_str = ",".join(tags2)
logo = c["logo"]
if logo:
logo = logo.replace("https://", "http://")
establish_date = None
if c.has_key("startDate"):
d = time.localtime(c["startDate"] / 1000)
if d.tm_year > 1980:
establish_date = datetime.datetime(d.tm_year, d.tm_mon, d.tm_mday)
address1 = None
address2 = None
if c.has_key("address1"):
address1 = c["address1"]
if c.has_key("address2"):
address2 = c["address2"]
location_id = 0
if address2 != None:
city = kr36_cities.get(str(address2), None)
if city != None:
location = parser_db_util.get_location(formCityName(city))
if location != None:
location_id = location["locationId"]
if location_id == 0 and address1 != None:
city = kr36_cities.get(str(address1), None)
if city != None:
location = parser_db_util.get_location(formCityName(city))
if location != None:
location_id = location["locationId"]
#logger.info("locationid =%s",location_id)
fullName = c["fullName"]
fullName = fullName.replace("_", "")
idx = fullName.rfind(u"公司")
if idx != -1:
fullName = fullName[:(idx + len(u"公司"))]
fullName = name_helper.company_name_normalize(fullName)
desc = ""
productDesc = None
modelDesc = None
operationDesc = None
teamDesc = None
marketDesc = None
compititorDesc = None
advantageDesc = None
planDesc = None
if c.has_key("projectAdvantage"):
productDesc = c["projectAdvantage"].strip()
if c.has_key("dataLights"):
operationDesc = c["dataLights"].strip()
if c.has_key("projectPlan"):
modelDesc = c["projectPlan"].strip()
if c.has_key("competitor"):
compititorDesc = c["competitor"].strip()
if c.has_key("intro"):
desc = c["intro"].strip()
if c.has_key("story"):
teamDesc = c["story"].strip()
return {
"status": c["status"],
"name": c["name"],
"fullName": fullName,
"description": desc,
"productDesc": productDesc,
"modelDesc": modelDesc,
"operationDesc": operationDesc,
"teamDesc": teamDesc,
"marketDesc": marketDesc,
"compititorDesc": compititorDesc,
"advantageDesc": advantageDesc,
"planDesc": planDesc,
"brief": c["brief"],
"round": 0,
"roundDesc": None,
"companyStatus": 2010,
'fundingType': 0,
"locationId": location_id,
"address": None,
"phone": None,
"establishDate": establish_date,
"logo": logo,
"source": SOURCE,
"sourceId": company_key,
"field": c.get("industry"),
"subField": None,
"tags": tags_str,
"headCountMin": None,
"headCountMax": None
}
def parse_company(item):
logger.info("parse_company")
company_key = item["key"]
#company basic info
c = item["content"]["company_base"]["data"]
#check if page is under development or is completed(CREATED)
# if c["status"] == "INIT":
# return {
# "status":c["status"],
# }
tags = item["content"]["company_base"]["data"]["industryTag"]
tags2 = []
for tag in tags:
tags2.append(tag["name"])
tags_str = ",".join(tags2)
logo = c["logo"]
if logo:
logo = logo.replace("https://", "http://")
establish_date = None
if c.has_key("startDate"):
d = time.localtime(c["startDate"] / 1000)
if d.tm_year > 1980:
establish_date = datetime.datetime(d.tm_year, d.tm_mon, d.tm_mday)
address1 = None
address2 = None
if c.has_key("address1"):
address1 = c["address1"]
if c.has_key("address2"):
address2 = c["address2"]
location_id = 0
if address2 != None:
city = kr36_cities.get(str(address2), None)
if city != None:
location = parser_db_util.get_location(formCityName(city))
if location != None:
location_id = location["locationId"]
if location_id == 0 and address1 != None:
city = kr36_cities.get(str(address1), None)
if city != None:
location = parser_db_util.get_location(formCityName(city))
if location != None:
location_id = location["locationId"]
#logger.info("locationid =%s",location_id)
fullName = c["fullName"]
fullName = fullName.replace("_", "")
idx = fullName.rfind(u"公司")
if idx != -1:
fullName = fullName[:(idx + len(u"公司"))]
fullName = name_helper.company_name_normalize(fullName)
desc = ""
productDesc = None
modelDesc = None
operationDesc = None
teamDesc = None
marketDesc = None
compititorDesc = None
advantageDesc = None
planDesc = None
otherDesc = None
if c.has_key("companyIntroduce"):
if c["companyIntroduce"]["productService"] is not None and c[
"companyIntroduce"]["productService"].strip(
) != "": # productService
productDesc = c["companyIntroduce"]["productService"]
if c["companyIntroduce"]["userMarket"] is not None and c[
"companyIntroduce"]["userMarket"].strip() != "":
marketDesc = c["companyIntroduce"]["userMarket"]
# if c.has_key("dataLights"): # 我们的用户
# operationDesc = c["dataLights"].strip()
# if c.has_key("projectPlan"): # 未来的我们
# modelDesc = c["projectPlan"].strip()
# if c.has_key("competitor"): # 与我们相似的产品
# compititorDesc = c["competitor"].strip()
if c.has_key("intro"): # 其他
# otherDesc = c["intro"].strip()
desc = c["intro"].strip()
# if c.has_key("story"): # 团队介绍
# teamDesc = c["story"].strip()
headCount = c["scale"].replace("人", "")
min_staff = None
max_staff = None
if headCount.strip() != "":
if headCount == "少于15":
min_staff = 1
max_staff = 15
else:
staffarr = headCount.split('-')
if len(staffarr) > 1:
try:
min_staff = int(staffarr[0])
max_staff = int(staffarr[1])
except:
pass
else:
try:
min_staff = int(staffarr[0].strip())
max_staff = None
except:
pass
return {
"name":
c["name"],
"fullName":
fullName if fullName is not None and fullName.strip() != "" else None,
"description":
desc,
"productDesc":
productDesc,
"modelDesc":
modelDesc,
"operationDesc":
operationDesc,
"teamDesc":
teamDesc,
"marketDesc":
marketDesc,
"compititorDesc":
compititorDesc,
"advantageDesc":
advantageDesc,
"planDesc":
planDesc,
"otherDesc":
otherDesc,
"brief":
c["brief"],
"round":
None,
"roundDesc":
None,
"companyStatus":
2010,
'fundingType':
0,
"locationId":
location_id,
"address":
None,
"phone":
None,
"establishDate":
establish_date,
"logo":
logo,
"source":
SOURCE,
"sourceId":
company_key,
"field":
None,
"subField":
None,
"tags":
tags_str,
"headCountMin":
min_staff,
"headCountMax":
max_staff
}
def parse_companyjobs(source_company_id, item, sourceId):
source_jobs = []
logger.info("source_company_id is %s", source_company_id)
if item["content"].has_key("version") and item["content"]["version"] == 2:
logger.info("version 2!!!!")
for type in item["content"]:
if type == "version":
continue
position_type = type
for jobpage in item["content"][type]:
jobs = jobpage['content']['data']['page']['result']
logger.info("%s has %s jobs", position_type, len(jobs))
for job_content in jobs:
#logger.info("%s has %s jobs", position_type, len(jobs))
if sourceId != str(job_content["companyId"]):
logger.info("sourceId is not correct")
continue
domain = 0
if position_type == '技术':
domain = 15010
elif position_type == '产品':
domain = 15020
elif position_type == '设计':
domain = 15030
elif position_type == '运营':
domain = 15040
elif position_type == '市场与销售':
domain = 15050
elif position_type == '职能':
domain = 15060
elif position_type == '金融':
domain = 15070
job_key = job_content["positionId"]
born_time = None
position = job_content.get("positionName")
education = job_content.get("education")
city = job_content.get("city")
salary = job_content["salary"]
work_year = job_content["workYear"]
update_time = job_content["createTime"]
location_id = 0
location_new = parser_db_util.get_location(city)
if location_new != None:
location_id = location_new["locationId"]
education_type = 0
if education == '大专':
education_type = 6020
elif education == '本科':
education_type = 6030
elif education == '硕士':
education_type = 6040
elif education == '博士':
education_type = 6050
workYear_type = 7000
if work_year == '应届毕业生':
workYear_type = 7010
elif work_year == '1年以下':
workYear_type = 7020
elif work_year == '1-3年':
workYear_type = 7030
elif work_year == '3-5年':
workYear_type = 7040
elif work_year == '5-10年':
workYear_type = 7050
elif work_year == '10年以上':
workYear_type = 7060
date = "%s" % time.strftime("%Y-%m-%d", time.localtime())
if '-' not in update_time and update_time != None:
update_time = date + ' ' + update_time.strip()
source_job = {
"sourceId": job_key,
"sourceCompanyId": source_company_id,
"position": position,
"salary": salary,
"description": None,
"domain": domain,
"locationId": location_id,
"educationType": education_type,
"workYearType": workYear_type,
"startDate": born_time,
"updateDate": update_time,
}
logger.info("job_key: %s", job_key)
# for i in source_job:
# logger.info("%s->%s",i,source_job[i])
source_jobs.append(source_job)
else:
jobs = item["content"]['content']['data']['page']['result']
for job_content in jobs:
job_key = job_content["positionId"]
born_time = job_content.get("bornTime")
position = job_content.get("positionName")
education = job_content.get("education")
city = job_content["city"]
# keywords = job_content["keyWords"]
salary = job_content["salary"]
work_year = job_content["workYear"]
position_type = job_content["positionFirstType"]
update_time = job_content["createTime"]
#location_id = parser_util.get_location_id(city)
location_id = 0
location_new = parser_db_util.get_location(city)
if location_new != None:
location_id = location_new["locationId"]
education_type = 0
if education == '大专':
education_type = 6020
elif education == '本科':
education_type = 6030
elif education == '硕士':
education_type = 6040
elif education == '博士':
education_type = 6050
workYear_type = 7000
if work_year == '应届毕业生':
workYear_type = 7010
elif work_year == '1年以下':
workYear_type = 7020
elif work_year == '1-3年':
workYear_type = 7030
elif work_year == '3-5年':
workYear_type = 7040
elif work_year == '5-10年':
workYear_type = 7050
elif work_year == '10年以上':
workYear_type = 7060
domain = 0
if position_type == '技术':
domain = 15010
elif position_type == '产品':
domain = 15020
elif position_type == '设计':
domain = 15030
elif position_type == '运营':
domain = 15040
elif position_type == '市场与销售':
domain = 15050
elif position_type == '职能':
domain = 15060
elif position_type == '金融':
domain = 15070
date = "%s" % time.strftime("%Y-%m-%d", time.localtime())
if '-' not in born_time:
born_time = date + ' ' + born_time.strip()
if '-' not in update_time and update_time != None:
update_time = date + ' ' + update_time.strip()
source_job = {
"sourceId": job_key,
"sourceCompanyId": source_company_id,
"position": position,
"salary": salary,
"description": None,
"domain": domain,
"locationId": location_id,
"educationType": education_type,
"workYearType": workYear_type,
"startDate": born_time,
"updateDate": update_time,
}
logger.info("job_key: %s", job_key)
#for i in source_job:
# logger.info("%s->%s",i,source_job[i])
source_jobs.append(source_job)
logger.info(
json.dumps(source_jobs, ensure_ascii=False, cls=util.CJsonEncoder))
logger.info("scid %s, sourceId %s currently has %s jobs",
source_company_id, sourceId, len(source_jobs))
return source_jobs
def parse_company(item):
if item is None:
return None
logger.info("*** base ***")
company_key = item["key"]
html = item["content"]
logger.info(company_key)
d = pq(html)
# logo_id processed in parser_db_util
'''
logo_id = None
if logo_url is not None:
logo_id = parser_util.get_logo_id(source, company_key, 'company', logo_url)
'''
if html.decode("utf-8").find("这个公司的主页还在建设") >= 0:
return {
"status": "No_Name",
}
name = d('.company_main > h1 > a').text()
link = d('.company_main > h1 > a').attr('href')
fullName = d('.company_main > h1 > a').attr('title')
fullName = name_helper.company_name_normalize(fullName)
if name is None or fullName is None or name.find("拉勾") >= 0:
return {
"status": "No_Name",
}
if len(name) > len(fullName):
name = fullName
if name is None or name.strip() == "":
name = fullName
chinese, companycheck = name_helper.name_check(fullName)
if companycheck is not True:
return {
"status": "No_Name",
}
logo = d('.top_info_wrap > img').attr('src')
if logo.startswith("http") or logo.startswith("https"):
pass
else:
logo = "http:" + logo
if logo.find("logo_default") >= 0:
logo = None
brief = d('.company_word').text()
desc_text = d('.company_intro_text').text()
if u"该公司尚未添加公司介绍" in desc_text or len(desc_text) < 10:
desc = None
else:
desc = d('.company_intro_text > .company_content').html()
desc = desc.replace('<span class="text_over">展开</span>', '')
soup = BeautifulSoup(desc, "lxml")
raw = soup.getText()
# logger.info(desc)
#logger.info(raw)
desc = raw
# if desc is None or desc.strip() == "":
# return {
# "status": "No_Name",
# }
field = ''
stage = ''
headCount = ''
location = ''
address = ''
try:
field = d(
'#basic_container > .item_content >ul > li:eq(0) > span').text()
stage = d(
'#basic_container > .item_content >ul > li:eq(1) > span').text()
headCount = d(
'#basic_container > .item_content >ul > li:eq(2) > span').text()
headCount = headCount[0:headCount.index(u'人')]
location = d(
'#basic_container > .item_content >ul > li:eq(3) > span').text()
address = d('.con_mlist_ul > li:eq(0) > p:eq(1)').text()
except:
pass
headCount = headCount.replace("people", "")
if headCount == "少于15":
min_staff = 1
max_staff = 15
else:
staffarr = headCount.split('-')
if len(staffarr) > 1:
min_staff = staffarr[0]
max_staff = staffarr[1]
else:
try:
min_staff = int(staffarr[0].strip())
max_staff = None
except:
min_staff = None
max_staff = None
funding_type = 0
if stage == '不需要融资':
stage = 0
funding_type = 8010
elif stage == '未融资':
stage = 0
elif stage == '天使轮':
stage = 1010
elif stage == 'A轮':
stage = 1030
elif stage == 'B轮':
stage = 1040
elif stage == 'C轮':
stage = 1050
elif stage == 'D轮及以上':
stage = 1060
elif stage == '上市公司':
stage = 1110
else:
stage = 0
location_id = 0
location_new = parser_db_util.get_location(location)
if location_new != None:
location_id = location_new["locationId"]
#website = util.norm_url(link)
website = url_helper.url_normalize(link)
logger.info("website: %s" % website)
artifacts = []
type, app_market, app_id = url_helper.get_market(website)
if type == 4010:
if item["url"] != website and website.find("lagou.com") == -1:
flag, domain = url_helper.get_domain(website)
if flag is not None:
if flag is False:
domain = None
artifacts.append({
"type": 4010,
"name": name,
"description": desc,
"link": website,
"domain": domain
})
elif type == 4020 or type == 4030:
domain = None
if domain is not None:
artifacts.append({
"type": type,
"name": name,
"description": desc,
"link": website,
"domain": domain
})
elif type == 4040:
domain = app_id
if domain is not None:
artifacts.append({
"type": 4040,
"name": name,
"description": desc,
"link": website,
"domain": domain
})
elif type == 4050:
domain = None
if app_market == 16010 or app_market == 16020:
android_app = parser_db_util.find_android_market(
app_market, app_id)
if android_app:
domain = android_app["apkname"]
else:
domain = app_id
if domain is not None:
artifacts.append({
"type": 4050,
"name": name,
"description": desc,
"link": website,
"domain": domain
})
source_company = {
"name":
name,
"fullName":
fullName if fullName is not None and fullName.strip() != "" else None,
"description":
desc,
"productDesc":
None,
"modelDesc":
None,
"operationDesc":
None,
"teamDesc":
None,
"marketDesc":
None,
"compititorDesc":
None,
"advantageDesc":
None,
"planDesc":
None,
"brief":
brief,
"round":
None,
"roundDesc":
None,
"companyStatus":
2010,
'fundingType':
funding_type,
"locationId":
location_id,
"address":
address,
"phone":
None,
"establishDate":
None,
"logo":
logo,
"source":
SOURCE,
"sourceId":
company_key,
"field":
field,
"subField":
None,
"tags":
None,
"headCountMin":
min_staff,
"headCountMax":
max_staff,
"artifacts":
artifacts,
"status":
1
}
return source_company
def parse_companyjobs(source_company_id, item, sourceId):
source_jobs = []
logger.info("source_company_id is %s", source_company_id)
if item["content"].has_key("version") and item["content"]["version"] == 2:
logger.info("version 2!!!!")
for ptype in item["content"]:
if ptype == "version":
continue
position_type = ptype
for content in item["content"][position_type]:
d = pq((html.fromstring(content.decode("utf-8"))))
# logger.info("this page has %s jobs", position_type, len(jobs))
domain = 0
if position_type == '技术':
domain = 15010
elif position_type == '产品':
domain = 15020
elif position_type == '设计':
domain = 15030
elif position_type == '运营':
domain = 15040
elif position_type == '市场' or position_type == '销售':
domain = 15050
elif position_type.find('职能') >= 0:
domain = 15060
elif position_type == '金融':
domain = 15070
for li in d('div.job-list> ul> li'):
dj = pq(li)
job_link = dj('a').eq(0).attr("href")
job_key = job_link.split("/")[-1].replace(".html", "")
born_time = dj('div.info-publis> p').text()
position = dj('div.title-box> div.job-title').text()
(city, work_year, education) = (None, None, None)
lll = dj(
'div.job-primary> div.info-primary> p').text().strip()
logger.info("lll %s", lll)
if len(lll.split(" ")) == 3:
education = lll.split(" ")[2]
city = lll.split(" ")[0]
work_year = lll.split(" ")[1]
logger.info("%s - %s - %s", city, work_year, education)
salary = dj('div.title-box> span').text()
# update_time = born_time
location_id = 0
location_new = parser_db_util.get_location(city)
if location_new != None:
location_id = location_new["locationId"]
education_type = 0
if education == '大专':
education_type = 6020
elif education == '本科':
education_type = 6030
elif education == '硕士':
education_type = 6040
elif education == '博士':
education_type = 6050
workYear_type = 7000
if work_year == '应届毕业生':
workYear_type = 7010
elif work_year == '1年以下':
workYear_type = 7020
elif work_year == '1-3年':
workYear_type = 7030
elif work_year == '3-5年':
workYear_type = 7040
elif work_year == '5-10年':
workYear_type = 7050
elif work_year == '10年以上':
workYear_type = 7060
t = datetime.datetime.today()
logger.info("born time: %s", born_time)
try:
if born_time.find("发布于") >= 0:
born_time = born_time.replace("发布于", "")
if born_time.find("月") >= 0:
logger.info("neeewdate:%s",
str(t.year) + "年" + born_time)
update_time = datetime.datetime.strptime(
str(t.year) + "年" + born_time, "%Y年%m月%d日")
if update_time > datetime.datetime.now():
update_time = update_time - datetime.timedelta(
days=365)
elif born_time.find("昨天") >= 0:
update_time = datetime.datetime.now(
) - datetime.timedelta(days=1)
else:
update_time = datetime.datetime.now()
else:
update_time = datetime.datetime.now()
except Exception, e:
logger.info(e)
update_time = datetime.datetime.now()
source_job = {
"source": 13055,
"sourceId": job_key,
"recruit_company_id": str(source_company_id),
"position": position,
"salary": salary,
"description": None,
"domain": domain,
"locationId": location_id,
"educationType": education_type,
"workYearType": workYear_type,
"startDate": update_time,
"updateDate": update_time,
"jobNature": None,
"positionAdvantage": None,
"companyLabelList": None,
"financeStage": None,
"district": None,
}
logger.info("job_key: %s", job_key)
for i in source_job:
logger.info("%s->%s", i, source_job[i])
source_jobs.append(source_job)
def parse_company(item):
# logger.info("parse_company")
d = pq(html.fromstring(item['content'].decode("utf-8")))
company_key = item["key"]
# company basic info
tags = []
for tag in d('.portfolio-user-tag .label').text().split():
if tag.strip() not in tags: tags.append(tag.strip())
tags_str = ",".join(tags)
logo = 'http:' + d('.portfolio-user-photo img').attr('src')
if logo:
logo = logo.replace("https://", "http://")
logo = logo.replace("@!msgs", "")
establish_date = None
companyName = d('.corp-name').text()
location_id = 0
city = d('.portfolio-user-tag').text().split(' ')[0]
if city != None: location = parser_db_util.get_location(city)
if location is None:
city = name_helper.get_location_from_company_name(companyName)[0]
if city != None:
location = parser_db_util.get_location(city)
if location != None:
location_id = location["locationId"]
# logger.info("locationid =%s",location_id)
fullName = companyName.replace("_", "")
fullName = name_helper.company_name_normalize(fullName)
# desc = d('.portfolio-corp p').text()
desc = d('.portfolio-user-bio .text').text()
productDesc = d('.portfolio-text').text()
website = d('.user-contact a').text()
if desc == '' or desc is None: desc = productDesc
shortName = d('.portfolio-user-info h1').text()
companyResult = {}
companyResult.update({
"name": shortName,
"fullName": fullName,
"description": desc,
"productDesc": productDesc,
"round": 0,
"roundDesc": None,
"companyStatus": 2010,
'fundingType': 0,
"locationId": location_id,
"address": None,
"phone": None,
"establishDate": None,
"logo": logo,
"source": SOURCE,
"sourceId": company_key,
"field": None,
"subField": None,
"tags": tags_str,
"headCountMin": None,
"headCountMax": None,
"brief": None,
"website": website,
})
return companyResult
def parse_company(item):
logger.info("parse_szse_stock")
# logger.info(item)
company_key = item["sourceId"]
#company basic info
c = item["baseinfo"]
try:
field = c["industry"].split(" ")[-1]
except:
field = None
result = parser_db_util.get_location(c["city"].replace("市",""))
if result != None:
location_id = result["locationId"]
else:
location_id = None
fullName = item["name"]
logger.info("parsing :%s|%s", fullName, company_key)
desc = None
brief = None
try:
desc = item["jqkaBrief"]["desc"]
except:
pass
try:
brief = item["jqkaBrief"]["brief"].split("。")[0]
logger.info(len(brief))
if len(brief) > 99:
brief = None
except:
pass
productDesc = None
modelDesc = None
operationDesc = None
teamDesc = None
marketDesc = None
compititorDesc = None
advantageDesc = None
planDesc = None
otherDesc = None
sourceId = item["stockwebsite"].split("=")[-1]
return {
"name": c["shortname"].replace("N",""),
"fullName": fullName.strip(),
"description": desc,
"productDesc": productDesc,
"modelDesc": modelDesc,
"operationDesc": operationDesc,
"teamDesc": teamDesc,
"marketDesc": marketDesc,
"compititorDesc": compititorDesc,
"advantageDesc": advantageDesc,
"planDesc": planDesc,
"otherDesc": otherDesc,
"brief": brief,
"round": 0,
"roundDesc": None,
"companyStatus": 2010,
'fundingType': 0,
"locationId": location_id,
"address": c["regLocation"] if c["regLocation"] is not None and c["regLocation"].strip()!="" else None,
"phone": None,
"establishDate": None,
"logo": None,
"source": SOURCE,
"sourceId": sourceId,
"field": field,
"subField": None,
"tags": None,
"headCountMin": None,
"headCountMax": None,
"englishName": c["englishName"] if c.has_key("englishName") else None
}
def parse_company(item):
logger.info("parse_sse_stock")
# logger.info(item)
company_key = item["sourceId"]
#company basic info
c = item["baseinfo"]
field = c["CSRC_GREAT_CODE_DESC"]
result = parser_db_util.get_location(c["AREA_NAME_DESC"].replace("市", ""))
if result != None:
location_id = result["locationId"]
else:
location_id = None
fullName = c["FULLNAME"]
logger.info("parsing :%s|%s", fullName, company_key)
desc = None
brief = None
try:
desc = item["jqkaBrief"]["desc"]
except:
pass
try:
brief = item["jqkaBrief"]["brief"].split("。")[0]
logger.info(len(brief))
if len(brief) > 99:
brief = None
except:
pass
productDesc = None
modelDesc = None
operationDesc = None
teamDesc = None
marketDesc = None
compititorDesc = None
advantageDesc = None
planDesc = None
otherDesc = None
return {
"name":
c["shortname"],
"fullName":
fullName.strip(),
"description":
desc,
"productDesc":
productDesc,
"modelDesc":
modelDesc,
"operationDesc":
operationDesc,
"teamDesc":
teamDesc,
"marketDesc":
marketDesc,
"compititorDesc":
compititorDesc,
"advantageDesc":
advantageDesc,
"planDesc":
planDesc,
"otherDesc":
otherDesc,
"brief":
brief,
"round":
0,
"roundDesc":
None,
"companyStatus":
2010,
'fundingType':
0,
"locationId":
location_id,
"address":
c["COMPANY_ADDRESS"]
if c.has_key("COMPANY_ADDRESS") and c["COMPANY_ADDRESS"] is not None
and c["COMPANY_ADDRESS"].strip() != "" else None,
"phone":
str(c["REPR_PHONE"])
if c.has_key("COMPANY_ADDRESS") and c["REPR_PHONE"] is not None
and str(c["REPR_PHONE"]).strip() not in ["", "-"] else None,
"establishDate":
None,
"logo":
None,
"source":
SOURCE,
"sourceId":
company_key,
"field":
field,
"subField":
None,
"tags":
None,
"headCountMin":
None,
"headCountMax":
None,
"englishName":
c["FULL_NAME_IN_ENGLISH"]
if c.has_key("FULL_NAME_IN_ENGLISH") else None
}
def parse_base(item):
if item is None:
return None
logger.info("*** base ***")
company_key = item["key"]
html = item["content"]
#logger.info(html)
d = pq(html)
company_short_name = ""
product_name = d('div.line-title> span> h1').clone().children().remove().end().text().strip()
if product_name is None or product_name.strip() == "":
product_name = d('div.line-title> span> b').clone().children().remove().end().text().strip()
temps = product_name.split("/",1)
if len(temps) == 2:
product_name = temps[0].strip()
company_short_name = temps[1].strip()
if company_short_name == "":
company_short_name = product_name
logger.info("product name: %s" % product_name)
logger.info("company short name: %s" % company_short_name)
company_name = d('div.des-more> div').eq(0).text().strip().replace("公司全称：","")
if company_name == "暂无" or company_name == "暂未收录":
company_name = ""
if company_name is None or company_name.strip() == "":
try:
company_name = d('div.des-more> h2').text().strip()
except:
pass
if company_name == "暂无" or company_name == "暂未收录":
company_name = ""
company_name = name_helper.company_name_normalize(company_name)
logger.info("company name: %s" % company_name)
if company_short_name == "" and company_name == "":
return
establish_date = None
str = d('div.des-more> div').eq(1).text().strip().replace("成立时间：","")
result = util.re_get_result('(\d*)\.(\d*)',str)
if result != None:
(year, month) = result
try:
if int(month) > 12:
month = "1"
except:
month = "1"
establish_date = datetime.datetime.strptime("%s-%s-1" % (year,month), '%Y-%m-%d')
logger.info("establish date: %s" % establish_date)
locationId=0
str = d('span.loca').text().strip()
#logger.info(str)
result = util.re_get_result(u'(.*?)·(.*?)$',str)
if result != None:
(province, city) = result
province = province.strip()
city = city.strip()
logger.info("location: %s-%s" % (province, city))
locationId = 0
result = parser_db_util.get_location(city)
if result != None:
locationId = result["locationId"]
else:
result = parser_db_util.get_location(province)
if result != None:
locationId = result["locationId"]
if locationId == 0:
loc1,loc2 = name_helper.get_location_from_company_name(company_name)
if loc1 is not None:
result = parser_db_util.get_location(loc1)
if result != None:
locationId = result["locationId"]
logger.info("locationId: %d" % locationId)
company_status = 2010
str = d('div.des-more> div').eq(2).text().strip()
if str == "已关闭":
company_status = 2020
logger.info("company_status: %d" % company_status)
funding_type = 0
str = d("span.tag.bg-c").text().strip()
logger.info("融资需求: %s" % str)
if str == "融资需求 · 需要融资":
funding_type = 8020
elif str == "融资需求 · 寻求收购":
funding_type = 8020
logger.info("funding_type=%d" % funding_type)
try:
brief = d("h2.seo-slogan").text().strip()
except:
brief = ""
logger.info("brief: %s" % brief)
if brief.find("暂未收录"):
brief = ""
field = d("span.scope.c-gray-aset> a").eq(0).text().strip()
logger.info("field: %s" % field)
sub_field = d("span.scope.c-gray-aset> a").eq(1).text().strip()
logger.info("sub field: %s" % sub_field)
tags = d("div.tagset.dbi.c-gray-aset> a >span").text().strip().replace(" ",",")
logger.info("tags: %s" % tags)
desc = d("div.des,div.desc,div.introduction,div.abstract,div.summary").text().\
replace("购买数据请联系","").replace('*****@*****.**',"").replace("itjuzi是一家数据服务公司","").strip()
logger.info("********desc: %s" % desc)
#logo
logo = d("div.pic >img").attr("src")
#if logo:
# logo = logo.replace("http://", "https://")
logger.info("logo: %s", logo)
# website = d('div.link-line> a').text().strip()
# if website is None or website == "":
# website = d('div.link-line> a.webTink').text().strip()
# if website is None or website == "":
# try:
# logger.info("here")
# website = d('div.link-line> span.weblink> a').eq(1).text().strip()
# logger.info(website)
# except:
# pass
artifacts = []
for ty in [1,2,3]:
if ty == 1:
was = d('div.link-line> a')
else:
was = d('div.link-line> span.weblink,span.webTink> a')
for wa in was:
webs =[]
try:
website = pq(wa).attr("href").strip()
if website=="http://%e6%9a%82%e6%97%a0" or website == "http://tt":
website = ""
website = url_helper.url_normalize(website)
logger.info("website: %s" % website)
webs.append(website)
# else:
# website = pq(wa).text().strip()
except:
pass
try:
website = pq(wa).text().strip()
if website == "http://%e6%9a%82%e6%97%a0" or website == "http://tt":
website = ""
website = url_helper.url_normalize(website)
logger.info("website: %s" % website)
webs.append(website)
# else:
# website = pq(wa).text().strip()
except:
pass
#
# if website=="http://%e6%9a%82%e6%97%a0":
# website = ""
# website = url_helper.url_normalize(website)
# logger.info("website: %s" % website)
# artifacts = []
for website in webs:
type, app_market, app_id = url_helper.get_market(website)
if type == 4010:
flag, domain = url_helper.get_domain(website)
if flag is not None:
if flag is False:
domain = None
artifacts.append({
"type":4010,
"name":product_name,
"desc":desc,
"link":website,
"domain": domain
})
elif type == 4020:
domain = app_id
if domain is not None:
artifacts.append({
"type": 4020,
"name": product_name,
"desc": None,
"link": website,
"domain": website
})
elif type == 4030:
domain = app_id
if domain is not None:
artifacts.append({
"type": 4030,
"name": product_name,
"desc": None,
"link": website,
"domain": None
})
elif type == 4040:
domain = app_id
if domain is not None:
artifacts.append({
"type":4040,
"name":product_name,
"desc":desc,
"link":website,
"domain": domain
})
elif type == 4050:
domain = None
if app_market == 16010 or app_market == 16020:
android_app = parser_db_util.find_android_market(app_market, app_id)
if android_app:
domain = android_app["apkname"]
else:
domain = app_id
if domain is not None:
artifacts.append({
"type":4050,
"name":product_name,
"desc":desc,
"link":website,
"domain": domain
})
#获投状态
roundStr = d('span.t-small.c-green').text().replace("(","").replace(")","").replace("获投状态：","").strip()
fundingRound,roundStr = itjuzi_helper.getFundingRound(roundStr)
logger.info("获投状态: %d, %s", fundingRound, roundStr)
logger.info("")
return {
"shortName": company_short_name,
"fullName": company_name if company_name is not None and company_name.strip() != "" else None,
"productName": product_name,
"description": desc,
"brief": brief,
"round": fundingRound,
"roundDesc": roundStr,
"companyStatus": company_status,
"fundingType": funding_type,
"locationId": locationId,
"establishDate": establish_date,
"logo": logo,
"sourceId": company_key,
"field": field,
"subField": sub_field,
"tags": tags,
"type":41010,
"artifacts":artifacts
}