def new_note(): if request.method == "POST": url = upload_s3(request) code = random_str() while True: row = c.execute("SELECT * FROM notes WHERE note_id=:id", { "id": code }).fetchall() if len(row) == 0: break code = random_str() c.execute( "INSERT INTO notes (content, status, img_url, title, description, note_id, user_id) VALUES (:content, :status, :img, :title, :desc, :note_id, :user_id)", { "content": request.form.get("content"), "status": request.form.get("status"), "img": url, "title": request.form.get("title"), "desc": request.form.get("desc"), "note_id": code, "user_id": session.get("user_id") }) conn.commit() return redirect(f"/n/{code}") else: return render_template("new.html", action="/new")
def create_requests_and_save_datas(douban_id): session = sqla['session'] cookies['bid'] = random_str(11) video = session.query(models.Video).filter_by(douban_id=douban_id).one() cover_url = video.cover covers_url = video.covers thumbnail_covers_url = video.thumbnail_covers photos_url = video.photos thumbnail_photos_url = video.thumbnail_photos wallpapers_url = video.wallpapers thumbnail_wallpapers_url = video.thumbnail_wallpapers down( cover_url, cookies, os.path.join(base_path, 'cover'), str(douban_id)+'_'+cover_url.split('/')[-1].strip('?') ) create_down(covers_url, douban_id, 'covers') create_down(thumbnail_covers_url, douban_id, 'thumbnail_covers') create_down(photos_url, douban_id, 'photos') create_down(thumbnail_photos_url, douban_id, 'thumbnail_photos') create_down(wallpapers_url, douban_id, 'wallpapers') create_down(thumbnail_wallpapers_url, douban_id, 'thumbnail_wallpapers')
def create_requests_and_save_datas(douban_id): session = sqla['session'] cookies['bid'] = random_str(11) r = requests.get( douban_celebrity_url + str(douban_id), cookies=cookies, timeout=5 ) if r.status_code != 200: return data = parsers.celebrity.start_parser(r.text) celebrity = session.query(models.Celebrity).filter_by( douban_id=douban_id ).one() for key in list(data.keys()): if type(data[key]) == list: data[key] = str(data[key]) for k, v in data.items(): setattr(celebrity, k, v) celebrity.is_detail = True session.commit() print(' '.join( ['celebrity', douban_id, data['name']] ))
def create_requests_and_save_datas(type, tag, sort): session = sqla['session'] cookies['bid'] = random_str(11) params = { 'type': type, 'tag': tag, 'sort': sort, 'page_limit': 2000, 'page_start': 0 } r = requests.get(douban_movie_api_url, params=params, cookies=cookies, timeout=20) if r.status_code != 200: return datas = parsers.douban_api.start_parser(r.text) for data in datas: douban_id = data.get('douban_id') if douban_id in movie_douban_ids: continue data['subtype'] = type data['crawler_tag'] = tag data['crawler_sort'] = sort movie = models.Movie(**data) session.add(movie) session.commit() movie_douban_ids.add(douban_id) print(','.join( [douban_id ,data.get('title')] ))
def create_requests_and_save_datas(douban_id): session = sqla['session'] cookies['bid'] = random_str(11) r = requests.get(douban_celebrity_url + str(douban_id), cookies=cookies, timeout=5) if r.status_code != 200: return data = parsers.celebrity.start_parser(r.text) celebrity = session.query( models.Celebrity).filter_by(douban_id=douban_id).one() for key in list(data.keys()): if type(data[key]) == list: data[key] = str(data[key]) for k, v in data.items(): setattr(celebrity, k, v) celebrity.is_detail = True session.commit() print(' '.join(['celebrity', douban_id, data['name']]))
def create_requests_and_save_datas(douban_id): session = sqla['session'] cookies['bid'] = random_str(11) celebrity = session.query(models.Celebrity).filter_by(douban_id=douban_id).one() cover_url = celebrity.cover thumbnail_cover_url = celebrity.thumbnail_cover photos_url = celebrity.photos thumbnail_photos_url = celebrity.thumbnail_photos down( cover_url, cookies, os.path.join(base_path, 'cover'), str(douban_id)+'_'+cover_url.split('/')[-1].strip('?') ) down( thumbnail_cover_url, cookies, os.path.join(base_path, 'thumbnail_cover'), str(douban_id)+'_'+cover_url.split('/')[-1].strip('?') ) create_down(photos_url, douban_id, 'photos') create_down(thumbnail_photos_url, douban_id, 'thumbnail_photos')
def create_down(str_urls, douban_id, category): urls = ast.literal_eval(str_urls or "[]") path = os.path.join(base_path, category) for url in urls: filename = str(douban_id) + '_' + url.split('/')[-1].strip('?') cookies['bid'] = random_str(11) down(url, cookies, path, filename)
def applications(): if request.method == "POST": img = upload_s3(request) client_id = random_str(25) client_secret = random_str(40) while True: _ = c.execute( "SELECT * FROM apps WHERE client_id=:id OR client_secret=:sec", { "id": client_id, "sec": client_secret }).fetchall() if len(_) == 0: break client_id = random_str(25) client_secret = random_str(25) c.execute( "INSERT INTO apps (user_id, name, img, client_id, client_secret, redirect_uri) VALUES (:id, :name, :img, :cid, :sec, :uri)", { "id": session.get("user_id"), "name": request.form.get("name"), "img": img, "cid": client_id, "sec": client_secret, "uri": request.form.get("uri") }) conn.commit() return redirect("/app") else: apps = c.execute("SELECT * FROM apps WHERE user_id=:user_id", { "user_id": session.get("user_id") }).fetchall() return render_template("app.html", apps=apps)
def train(request): data = request.get_json() gd = fetch_gesture(data) if 'name' not in data: raise ServerError('No class provided', 400) if data['name'] not in Classifier.classes: raise ServerError('Invalid class', 400) gd.draw_pos() image_path = gd.pos_to_image(dir='data/imgs/' + data['name'], filename='img_' + random_str()) gesture = Gesture(gd, image_path) return {'success': True, 'gesture': gesture.to_dict()}
def parse(filename, outputdir='data/imgs'): with open(filename) as json_file: data = json.load(json_file) cur = 1 prefix = random_str() for gdata in data: if len(gdata) < 20: continue gd = GestureData(gdata) gd.normalize() gd.filter() gd.find_average() gd.pos_to_image(dir=outputdir, filename='img_' + prefix + '_' + str(cur)) cur += 1
def register(request): if request.method == "POST": fname = request.POST["first-name"] lname = request.POST["last-name"] email = request.POST["email"] country_code = request.POST["country-code"] phone = request.POST["phone-number"] password = request.POST["password"] role = request.POST["role"] address = request.POST["address"] address2 = request.POST.get("address2") city = request.POST["city"] state = request.POST["state"] country = request.POST["country"] zip = request.POST["zip"] User.objects.create_user(username=email, email=email, first_name=fname, last_name=lname, password=password).save() u = User.objects.get(username=email) if role == "shopper": role = 0 else: role = 1 Profile(user_id=u.id, role=role, address=address + " " + address2, city=city, state=state, country=country, zip=zip).save() code = random_str() c.messages.create(from_='+19162800623', body='TWOFA Code: ' + code, to='+' + country_code + phone) TwoFAToken(user_id=u.id, code=code, phone='+' + country_code + phone).save() return HttpResponse( "registered successfully, please check your phone to complete 2FA") else: return render(request, "authentication/register.html")
def create_requests_and_save_datas(bilibili_id): session = sqla['session'] cookies['sid'] = random_str(8) r = requests.get( bilibili_animation_url + str(bilibili_id), cookies=cookies, timeout=10 ) if r.status_code != 200: return data = parsers.animation.start_parser(r.text) animation = session.query(models.Animation).filter_by( bilibili_id=bilibili_id ).one() genre_class = models.AnimationGenre for k, v in data.items(): if k == 'genres': for genre in v: try: genre_obj = genre_class(**genre) session.add(genre_obj) session.commit() except (IntegrityError, InvalidRequestError): session.rollback() genre_obj = session.query(genre_class).filter_by( name=genre['name'] ).one() animation.genres.append(genre_obj) for k, v in data.items(): if k != 'genres': if type(v) == list: v = str(v) setattr(animation, k, v) animation.is_detail = True session.commit() print(','.join( [bilibili_id, data.get('title')] ))
def create_requests_and_save_datas(douban_id): session = sqla['session'] cookies['bid'] = random_str(11) celebrity = session.query( models.Celebrity).filter_by(douban_id=douban_id).one() cover_url = celebrity.cover thumbnail_cover_url = celebrity.thumbnail_cover photos_url = celebrity.photos thumbnail_photos_url = celebrity.thumbnail_photos down(cover_url, cookies, os.path.join(base_path, 'cover'), str(douban_id) + '_' + cover_url.split('/')[-1].strip('?')) down(thumbnail_cover_url, cookies, os.path.join(base_path, 'thumbnail_cover'), str(douban_id) + '_' + cover_url.split('/')[-1].strip('?')) create_down(photos_url, douban_id, 'photos') create_down(thumbnail_photos_url, douban_id, 'thumbnail_photos')
def create_requests_and_save_datas(page): session = sqla['session'] cookies['sid'] = random_str(8) params = { 'app': 'bangumi', 'page': page, 'indexType': 0, 'pagesize': 30, 'action': 'site_season_index' } r = requests.get( bilibili_api_url, params=params, cookies=cookies, timeout=10 ) if r.status_code != 200: return text = r.text if text == 'null': return datas = parsers.bilibili_api.start_parser(text) for data in datas: bilibili_id = data.get('bilibili_id') if bilibili_id in animation_bilibili_ids: continue try: t = session.query(models.Animation).filter( models.Animation.title.like('%'+data['title']+'%') ).one() except NoResultFound: animation = models.Animation(**data) session.add(animation) session.commit() print(','.join( [data.get('bilibili_id'), data.get('title')] )) except MultipleResultsFound: pass
def predict(request): data = request.get_json() gd = fetch_gesture(data) image_path = gd.pos_to_image(dir='data/tmp', filename='img_' + random_str()) prediction = classifier.predict(image_path) idx = np.unravel_index(np.argmax(prediction, axis=None), prediction.shape) gesture = Gesture(gd, image_path, Classifier.classes[idx[1]]) return { 'success': True, 'prediction': [*zip(map(float, prediction.tolist()[0]), Classifier.classes)], 'gesture': gesture.to_dict() }
def create_requests_and_save_datas(bilibili_id): session = sqla['session'] cookies['sid'] = random_str(8) r = requests.get(bilibili_animation_url + str(bilibili_id), cookies=cookies, timeout=10) if r.status_code != 200: return data = parsers.animation.start_parser(r.text) animation = session.query( models.Animation).filter_by(bilibili_id=bilibili_id).one() genre_class = models.AnimationGenre for k, v in data.items(): if k == 'genres': for genre in v: try: genre_obj = genre_class(**genre) session.add(genre_obj) session.commit() except (IntegrityError, InvalidRequestError): session.rollback() genre_obj = session.query(genre_class).filter_by( name=genre['name']).one() animation.genres.append(genre_obj) for k, v in data.items(): if k != 'genres': if type(v) == list: v = str(v) setattr(animation, k, v) animation.is_detail = True session.commit() print(','.join([bilibili_id, data.get('title')]))
def create_requests_and_save_datas(page): session = sqla['session'] cookies['sid'] = random_str(8) params = { 'app': 'bangumi', 'page': page, 'indexType': 0, 'pagesize': 30, 'action': 'site_season_index' } r = requests.get(bilibili_api_url, params=params, cookies=cookies, timeout=10) if r.status_code != 200: return text = r.text if text == 'null': return datas = parsers.bilibili_api.start_parser(text) for data in datas: bilibili_id = data.get('bilibili_id') if bilibili_id in animation_bilibili_ids: continue try: t = session.query(models.Animation).filter( models.Animation.title.like('%' + data['title'] + '%')).one() except NoResultFound: animation = models.Animation(**data) session.add(animation) session.commit() print(','.join([data.get('bilibili_id'), data.get('title')])) except MultipleResultsFound: pass
def create_requests_and_save_datas(douban_id): session = sqla['session'] cookies['bid'] = random_str(11) video = session.query(models.Video).filter_by(douban_id=douban_id).one() cover_url = video.cover covers_url = video.covers thumbnail_covers_url = video.thumbnail_covers photos_url = video.photos thumbnail_photos_url = video.thumbnail_photos wallpapers_url = video.wallpapers thumbnail_wallpapers_url = video.thumbnail_wallpapers down(cover_url, cookies, os.path.join(base_path, 'cover'), str(douban_id) + '_' + cover_url.split('/')[-1].strip('?')) create_down(covers_url, douban_id, 'covers') create_down(thumbnail_covers_url, douban_id, 'thumbnail_covers') create_down(photos_url, douban_id, 'photos') create_down(thumbnail_photos_url, douban_id, 'thumbnail_photos') create_down(wallpapers_url, douban_id, 'wallpapers') create_down(thumbnail_wallpapers_url, douban_id, 'thumbnail_wallpapers')
def create_requests_and_save_datas(type, tag, sort): session = sqla['session'] cookies['bid'] = random_str(11) params = { 'type': type, 'tag': tag, 'sort': sort, 'page_limit': 2000, 'page_start': 0 } r = requests.get(douban_movie_api_url, params=params, cookies=cookies, timeout=20) if r.status_code != 200: return datas = parsers.douban_api.start_parser(r.text) for data in datas: douban_id = data.get('douban_id') if douban_id in video_douban_ids: continue data['subtype'] = type data['crawler_tag'] = tag data['crawler_sort'] = sort if type == 'movie': video = models.Movie(**data) elif type == 'tv' and tag == '日本动画': video = models.Animation(**data) else: video = models.TV(**data) session.add(video) session.commit() video_douban_ids.add(douban_id) print(','.join([douban_id, data.get('title')]))
def access_token(): _app = c.execute( "SELECT * FROM apps WHERE client_id=:id AND client_secret=:sec", { "id": request.form.get("client_id"), "sec": request.form.get("client_secret") }).fetchall() code = c.execute("SELECT * FROM oauth_codes WHERE code=:c", { "c": request.form.get("code") }).fetchall() if len(code) == 0: return "INVALID CODE" if len(_app) == 0: return "INVALID CLIENT ID/CLIENT SECRET!" while True: token = random_str(70) _ = c.execute("SELECT * FROM tokens WHERE token=:t", { "t": token }).fetchall() if len(_) == 0: break c.execute("DELETE FROM oauth_codes WHERE code=:code", {"code": request.form.get("code")}) conn.commit() c.execute( "INSERT INTO tokens (app_id, user_id, token) VALUES (:a, :u, :t)", { "a": _app[0][0], "u": code[0][1], "t": token }).fetchall() conn.commit() return token
def twofa_verify(request: HttpRequest): if request.method == "POST": code = request.POST["code"] try: t = TwoFAToken.objects.get(code=code) except: return HttpResponse("Invalid code") user = User.objects.get(id=t.user_id) AuthorizedDevice(user_id=user.id).save() a = AuthorizedDevice.objects.filter(user_id=user.id)[::-1][0] login(request, user) request.session["deviceid"] = str(a.uuid) return HttpResponseRedirect("/auth/login/") else: code = random_str() TwoFAToken(user_id=request.session.get("2fa_user_id"), code=code).save() phone = TWOFAVerified.objects.get( user_id=request.session.get("2fa_user_id")).phone c.messages.create(from_='+19162800623', body='TWOFA Code: ' + code, to=phone) return render(request, "authentication/2fa.html")
def create_requests_and_save_datas(douban_id): session = sqla['session'] cookies['bid'] = random_str(11) r = requests.get(douban_movie_url + str(douban_id), cookies=cookies, timeout=10) if r.status_code != 200: return data = parsers.movie.start_parser(r.text) data['douban_url'] = r.url directors = data.pop('directors', []) director_douban_ids = set(director['douban_id'] for director in directors) playwrights = data.pop('playwrights', []) playwright_douban_ids = set(playwright['douban_id'] for playwright in playwrights) actors = data.pop('actors', []) actor_douban_ids = set(actor['douban_id'] for actor in actors) celebrities = directors + playwrights + actors celebrity_douban_ids = \ director_douban_ids | playwright_douban_ids | actor_douban_ids douban_id_celebrity_obj_dict = {} for celebrity in celebrities: celebrity_douban_id = celebrity['douban_id'] if celebrity_douban_id is not None: try: celebrity_obj = models.Celebrity(**celebrity) session.add(celebrity_obj) session.commit() except (IntegrityError, InvalidRequestError): session.rollback() celebrity_obj = session.query(models.Celebrity).filter_by( douban_id=celebrity_douban_id).first() douban_id_celebrity_obj_dict[celebrity_douban_id] = celebrity_obj video = session.query(models.Video).filter_by(douban_id=douban_id).one() video.directors.clear() video.playwrights.clear() video.actors.clear() for (celebrity_douban_id, celeBrity_obj) in douban_id_celebrity_obj_dict.items(): if celebrity_douban_id in director_douban_ids: video.directors.append(celebrity_obj) if celebrity_douban_id in playwright_douban_ids: video.playwrights.append(celebrity_obj) if celebrity_douban_id in actor_douban_ids: video.actors.append(celebrity_obj) session.commit() """If use query.update(data), an error is raised, beacuse movie table is multiple table and we want to update movie table and subject table some columns. """ video.genres.clear() video.countries.clear() video.languages.clear() session.commit() table_name = video.__tablename__ if table_name == 'movies': genre_class = models.MovieGenre elif table_name == 'tvs': genre_class = models.TVGenre elif table_name == 'animations': genre_class = models.AnimationGenre for k, v in data.items(): if k == 'genres': for genre in v: try: genre_obj = genre_class(**genre) session.add(genre_obj) session.commit() except (IntegrityError, InvalidRequestError): session.rollback() genre_obj = session.query(genre_class).filter_by( name=genre['name']).one() video.genres.append(genre_obj) elif k == 'countries': for country in v: try: country_obj = models.Country(**country) session.add(country_obj) session.commit() except (IntegrityError, InvalidRequestError): session.rollback() country_obj = session.query( models.Country).filter_by(name=country['name']).one() video.countries.append(country_obj) elif k == 'languages': for language in v: try: language_obj = models.Language(**language) session.add(language_obj) session.commit() except (IntegrityError, InvalidRequestError): session.rollback() language_obj = session.query(models.Language).filter_by( name=language['name']).one() video.languages.append(language_obj) session.commit() '''Why set other value not in above for cycle? Beacuse above "for cycle" have rollback. ''' for k, v in data.items(): if k != 'genres' and k != 'countries' and k != 'languages': if k == 'aliases' or k == 'thumbnail_photos': v = str(v) setattr(video, k, v) session.commit() # parser movie photo r = requests.get(douban_movie_url + str(douban_id) + '/all_photos', cookies=cookies, timeout=10) photo_data = parsers.movie_photo.start_parser(r.text) for k, v in photo_data.items(): v = str(v) setattr(video, k, v) video.is_detail = True session.commit() print(','.join([table_name, douban_id, data.get('title')]))
def create_requests_and_save_datas(douban_id): session = sqla['session'] cookies['bid'] = random_str(11) r = requests.get( douban_movie_url + str(douban_id), cookies=cookies, timeout=10 ) if r.status_code != 200: return data = parsers.movie.start_parser(r.text) data['douban_url'] = r.url directors = data.pop('directors', []) director_douban_ids = set(director['douban_id'] for director in directors) playwrights = data.pop('playwrights', []) playwright_douban_ids = set( playwright['douban_id'] for playwright in playwrights ) actors = data.pop('actors', []) actor_douban_ids = set(actor['douban_id'] for actor in actors) celebrities = directors + playwrights + actors celebrity_douban_ids = \ director_douban_ids | playwright_douban_ids | actor_douban_ids douban_id_celebrity_obj_dict = {} for celebrity in celebrities: celebrity_douban_id = celebrity['douban_id'] if celebrity_douban_id is not None: try: celebrity_obj = models.Celebrity(**celebrity) session.add(celebrity_obj) session.commit() except (IntegrityError, InvalidRequestError): session.rollback() celebrity_obj = session.query(models.Celebrity).filter_by( douban_id=celebrity_douban_id ).first() douban_id_celebrity_obj_dict[celebrity_douban_id] = celebrity_obj video = session.query(models.Video).filter_by(douban_id=douban_id).one() video.directors.clear() video.playwrights.clear() video.actors.clear() for (celebrity_douban_id, celeBrity_obj) in douban_id_celebrity_obj_dict.items(): if celebrity_douban_id in director_douban_ids: video.directors.append(celebrity_obj) if celebrity_douban_id in playwright_douban_ids: video.playwrights.append(celebrity_obj) if celebrity_douban_id in actor_douban_ids: video.actors.append(celebrity_obj) session.commit() """If use query.update(data), an error is raised, beacuse movie table is multiple table and we want to update movie table and subject table some columns. """ video.genres.clear() video.countries.clear() video.languages.clear() session.commit() table_name = video.__tablename__ if table_name == 'movies': genre_class = models.MovieGenre elif table_name == 'tvs': genre_class = models.TVGenre elif table_name == 'animations': genre_class = models.AnimationGenre for k, v in data.items(): if k == 'genres': for genre in v: try: genre_obj = genre_class(**genre) session.add(genre_obj) session.commit() except (IntegrityError, InvalidRequestError): session.rollback() genre_obj = session.query(genre_class).filter_by( name=genre['name'] ).one() video.genres.append(genre_obj) elif k == 'countries': for country in v: try: country_obj = models.Country(**country) session.add(country_obj) session.commit() except (IntegrityError, InvalidRequestError): session.rollback() country_obj = session.query(models.Country).filter_by( name=country['name'] ).one() video.countries.append(country_obj) elif k == 'languages': for language in v: try: language_obj = models.Language(**language) session.add(language_obj) session.commit() except (IntegrityError, InvalidRequestError): session.rollback() language_obj = session.query(models.Language).filter_by( name=language['name'] ).one() video.languages.append(language_obj) session.commit() '''Why set other value not in above for cycle? Beacuse above "for cycle" have rollback. ''' for k, v in data.items(): if k != 'genres' and k != 'countries' and k != 'languages': if k == 'aliases' or k == 'thumbnail_photos': v = str(v) setattr(video, k, v) session.commit() # parser movie photo r = requests.get( douban_movie_url + str(douban_id) + '/all_photos', cookies=cookies, timeout=10 ) photo_data = parsers.movie_photo.start_parser(r.text) for k, v in photo_data.items(): v = str(v) setattr(video, k, v) video.is_detail = True session.commit() print(','.join( [table_name, douban_id, data.get('title')] ))
def oauth(): if request.method == "POST": client_id = request.args.get("client_id") app = c.execute("SELECT * FROM apps WHERE client_id=:id", { "id": client_id }).fetchall() c.execute("INSERT INTO user_app (user_id, app_id) VALUES (:u, :a)", { "u": session.get("user_id"), "a": app[0][0] }) conn.commit() code = random_str(50) while True: _ = c.execute("SELECT * FROM oauth_codes WHERE code=:code", { "code": code }).fetchall() if len(_) == 0: break code = random_str(50) c.execute( "INSERT INTO oauth_codes (app_id, user_id, code) VALUES (:id, :user, :code)", { "id": app[0][0], "user": session.get("user_id"), "code": code }) conn.commit() return redirect(f"{app[0][6]}?code={code}") client_id = request.args.get("client_id") app = c.execute("SELECT * FROM apps WHERE client_id=:id", { "id": client_id }).fetchall() if len(app) == 0: return "INVALID CLIENT ID" if session.get("user_id"): user_app = c.execute("SELECT * FROM user_app WHERE app_id=:app_id", { "app_id": app[0][0] }).fetchall() if len(user_app) == 0: return render_template("authorize.html", app=app) # generate code code = random_str(50) while True: _ = c.execute("SELECT * FROM oauth_codes WHERE code=:code", { "code": code }).fetchall() if len(_) == 0: break code = random_str(50) c.execute( "INSERT INTO oauth_codes (app_id, user_id, code) VALUES (:id, :user, :code)", { "id": app[0][0], "user": session.get("user_id"), "code": code }) conn.commit() return redirect(f"{app[0][6]}?code={code}") return redirect(f"/login?next=/authorize?client_id={client_id}")
def create_requests_and_save_datas(douban_id): session = sqla['session'] cookies['bid'] = random_str(11) r = requests.get(douban_movie_url + str(douban_id), cookies=cookies, timeout=10) if r.status_code != 200: return data = parsers.movie.start_parser(r.text) data['douban_url'] = r.url directors = data.pop('directors', []) director_douban_ids = set(director['douban_id'] for director in directors) playwrights = data.pop('playwrights', []) playwright_douban_ids = set(playwright['douban_id'] for playwright in playwrights) actors = data.pop('actors', []) actor_douban_ids = set(actor['douban_id'] for actor in actors) celebrities = directors + playwrights + actors celebrity_douban_ids = director_douban_ids | playwright_douban_ids | actor_douban_ids douban_id_celebrity_obj_dict = {} for celebrity in celebrities: celebrity_douban_id = celebrity['douban_id'] if celebrity_douban_id is not None: try: celebrity_obj = models.Celebrity(**celebrity) session.add(celebrity_obj) session.commit() except (IntegrityError, InvalidRequestError): session.rollback() celebrity_obj = session.query(models.Celebrity).filter_by(douban_id=celebrity_douban_id).first() douban_id_celebrity_obj_dict[celebrity_douban_id] = celebrity_obj movie = session.query(models.Movie).filter_by(douban_id=douban_id).one() movie.directors.clear() movie.playwrights.clear() movie.actors.clear() for celebrity_douban_id, celebrity_obj in douban_id_celebrity_obj_dict.items(): if celebrity_douban_id in director_douban_ids: movie.directors.append(celebrity_obj) if celebrity_douban_id in playwright_douban_ids: movie.playwrights.append(celebrity_obj) if celebrity_douban_id in actor_douban_ids: movie.actors.append(celebrity_obj) session.commit() # If use query.update(data), an error is raised, beacuse movie table is multiple table and we want to update movie table and subject table some columns. movie.genres.clear() movie.countries.clear() movie.languages.clear() session.commit() for k, v in data.items(): if k == 'genres': for movie_genre in v: try: movie_genre_obj = models.MovieGenre(**movie_genre) session.add(movie_genre_obj) session.commit() except (IntegrityError, InvalidRequestError): session.rollback() movie_genre_obj = session.query(models.MovieGenre).filter_by(name=movie_genre['name']).one() movie.genres.append(movie_genre_obj) elif k == 'countries': for movie_country in v: try: movie_country_obj = models.Country(**movie_country) session.add(movie_country_obj) session.commit() except (IntegrityError, InvalidRequestError): session.rollback() movie_country_obj = session.query(models.Country).filter_by(name=movie_country['name']).one() movie.countries.append(movie_country_obj) elif k == 'languages': for movie_language in v: try: movie_language_obj = models.Language(**movie_language) session.add(movie_language_obj) session.commit() except (IntegrityError, InvalidRequestError): session.rollback() movie_language_obj = session.query(models.Language).filter_by(name=movie_language['name']).one() movie.languages.append(movie_language_obj) session.commit() '''Why set other value not in above for cycle? Beacuse above "for cycle" have rollback. ''' for k, v in data.items(): if k!= 'genres' and k!='countries' and k!='languages': if k == 'aliases' or k == 'thumbnail_photos': v = str(v) setattr(movie, k, v) session.commit() # parser movie photo r = requests.get(douban_movie_url + str(douban_id) + '/all_photos', cookies=cookies, timeout=10) data = parsers.movie_photo.start_parser(r.text) for k, v in data.items(): v = str(v) setattr(movie, k, v) session.commit() print(','.join( ['movie', douban_id, movie.title] ))