def update_province_area_state(res):
"""
更新各省级数据
:return:
"""
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
data = session.query(Count).filter(Count.area_type == 'province').all()
map_data = {}
for count in data:
map_data[count.area_name] = count
for o in res:
comment = o['comment']
key = o['provinceShortName']
count = map_data.get(key)
if not count:
count = Count()
count.from_source = 'dxy'
count.area_name = key
count.area_type = 'province'
count.province = key
session.add(count)
json_to_model(count, o)
if comment != '':
num_data = format_data(comment)
filter_data(num_data, count)
count.update_time = now
session.commit()
def sync_area_data():
hubei_count = session.query(Count).filter(Count.area_name == '湖北').first()
beijing_count = session.query(Count).filter(
Count.area_name == '北京').first()
china_count = session.query(Count).filter(Count.area_name == '全国').first()
hubei = session.query(HubeiCount).first()
beijing = session.query(BeiJingCount).first()
china = session.query(ChinaCount).first()
sync_data(hubei, hubei_count)
sync_data(beijing, beijing_count)
sync_data(china, china_count)
session.commit()
def save_articles(data):
articles = session.query(Article).all()
map_data = []
for article in articles:
map_data.append(article.title)
for i in data:
if i['title'] not in map_data:
article = Article()
article_time = i['time']
article.time = time.strptime(article_time, "%Y.%m.%d%H:%M:%S")
article.detail = i['detail']
article.link = i['link']
article.title = i['title']
session.add(article)
session.commit()
def save_data(params, data):
for item in data:
migration = Migration()
migration.id = str(uuid.uuid4().hex)
migration.province_name = item['province_name']
migration.city_name = item['city_name']
migration.value = item['value']
migration.area_id = params['id']
migration.area_type = params['dt']
migration.area_name = params['area_name']
migration.create_date = params['date']
migration.migrate_type = params['type']
session.add(migration)
session.commit()
def save_country_data(res):
"""
保存全国的数据
:return:
"""
count = session.query(Count).filter(Count.area_name == '全国').first()
if not count:
count = Count()
count.area_type = 'country'
count.area_name = '全国'
count.from_source = 'dxy'
session.add(count)
filter_data(res, count)
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
count.update_time = now
session.commit()
def save_data(data):
videos = session.query(CCTVVideo).order_by(desc(CCTVVideo.date)).limit(5)
video_map = []
for item in videos:
video_map.append(item.source_url)
for i in data:
if i['wwwUrl'] in video_map:
continue
video = CCTVVideo()
video.title = i['title']
video.source_url = i['wwwUrl']
video.id = str(uuid.uuid4().hex)
timeNum = i['pubTime']
timeStamp = float(timeNum / 1000)
timeArray = time.localtime(timeStamp)
otherStyleTime = time.strftime("%Y-%m-%d %H:%M:%S", timeArray)
video.date = otherStyleTime
session.add(video)
session.commit()
def main():
print("--- Enter info ---")
site = input(
"Site (polovniautomobili.com, kupujemprodajem.com, nekretnine.rs): ")
if site not in SPIDERS_MAP.keys():
sys.exit("Entered site not available.")
url = input("URL of first page to crawl: ")
if site not in url:
sys.exit("Entered URL not valid.")
recipients = input("Recipients email (separated by comma for multi): ")
recipients = [validate_email(r.strip()) for r in recipients.split(",")]
name = input("Enter name/title of site settings(example: Ads for car): ")
s = Site(name=name, site=site, url=url, recipients=recipients)
session.add(s)
session.commit()
print(f'Site "{name}" added.')
print('Run "python run.py" to run worker to get new ads.')
def update_city_area_state(res):
data = session.query(Count).filter(Count.area_type == 'city').all()
map_data = {}
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
for count in data:
map_data["city_" + count.province + "_" + count.area_name] = count
for province in res:
cities = province['cities']
province = province['provinceShortName']
for city in cities:
city_key = city['cityName']
city_count = map_data.get("city_" + province + "_" + city_key)
if not city_count:
city_count = Count()
city_count.from_source = 'dxy'
city_count.area_name = city_key
city_count.area_type = 'city'
city_count.province = province
session.add(city_count)
json_to_model(city_count, city)
city_count.update_time = now
session.commit()
def save_record_data():
"""
保存对应记录
:return:
"""
counts = session.query(Count).all()
for count in counts:
count_record = CountRecord()
count_record.new_case = count.new_case
count_record.confirm_case = count.confirm_case
count_record.cure_case = count.cure_case
count_record.probable_case = count.probable_case
count_record.dead_case = count.dead_case
count_record.province = count.province
count_record.area_type = count.area_type
count_record.area_name = count.area_name
count_record.update_time = count.update_time
count_record.from_source = count.from_source
count_record.memo = count.memo
session.add(count_record)
session.commit()