def reupload(book_id, email="", key=""): subtitle = "Reupload book to Internet Archive" form = ReUpload() if email in (None, "") and key not in (None, ""): return render_template('reupload.html', subtitle=subtitle, form=form) if key != hashlib.md5(str(email) + str(keys.flask_app_secret)).hexdigest( ) and key not in (None, ""): return render_template('reupload.html', subtitle=subtitle, form=form) form = ReUpload() if form.validate() or key not in (None, ""): book_values = re.search("(.*):(.*)", book_id) library = book_values.group(1) Id = book_values.group(2) ia_identifier_suffix = get_valid_identifier_suffix(library, Id) reset_book_progress(library, ia_identifier_suffix) redis = redis_py.Redis() redis_key3 = keys.redis_key3 book_metadata = redis_py.get(redis_key3 + ":" + book_id + ":meta_data", True) book_key = "%s:%s:%s" % (redis_key3, library, ia_identifier_suffix) metadata_key = book_key + ":meta_data" book_request_key = book_key + ":requests" redis_py.set(metadata_key, book_metadata, True) request = dict(email=form.email.data) redis_py.sadd(book_request_key, json.dumps(request), request_cache=True) book = models.Book(book_id=ia_identifier_suffix, library=library, requests=json.dumps(request), meta_data=book_metadata) db.session.add(book) db.session.commit() redis_key1 = keys.redis_key1 q_global_job = redis_py.Queue(redis_key1 + "global") q_global_job.add( json.dumps(dict(library=library, book_id=ia_identifier_suffix))) q = redis_py.Queue(redis_key1) q.add( json.dumps( dict(library=library, Id=Id, ia_identifier_suffix=ia_identifier_suffix))) flash(error_msg(100, email=form.email.data)) return redirect( url_for('progress', book_id=library + ":" + ia_identifier_suffix)) else: return render_template('reupload.html', subtitle=subtitle, form=form)
def submit_job(stored_request): """Add book-request to the job queues""" redis_key1 = keys.redis_key1 lock_key1 = keys.lock_key1 q = redis_py.Queue(redis_key1) q_global_job = redis_py.Queue(redis_key1 + "global") redis = redis_py.Redis() redis_key3 = keys.redis_key3 book_key = "%s:%s:%s" % (redis_key3, stored_request.library, stored_request.book_id) book_request_key = book_key + ":requests" Lock = redis_py.Lock(lock_key1) locked = False if redis.exists(book_request_key): locked = Lock.acquire(timeout=60) request = dict(sno=stored_request.sno, email=stored_request.email) redis_py.sadd(book_request_key, json.dumps(request), request_cache=True) redundant_request = models.Request.query.filter_by( md5_book=stored_request.md5_book, confirmed=1, job_submitted=1).first() if redundant_request == None: reset_book_progress(stored_request.library, stored_request.book_id) md5_book = hashlib.md5(stored_request.book_id + stored_request.library).hexdigest() library_url_key = book_key + ":library_url" library_url = redis_py.get(library_url_key, True) metadata_key = book_key + ":meta_data" meta_data = redis_py.get(metadata_key, True) book = models.Book(book_id=stored_request.book_id, library=stored_request.library, md5_book=stored_request.md5_book, connected_request_sno=stored_request.sno, requests=json.dumps(request), library_url=library_url, meta_data=meta_data) db.session.add(book) #db.session.commit() q.add(stored_request.sno) stored_request.job_submitted = 1 db.session.commit() q_global_job.add( json.dumps( dict(library=stored_request.library, book_id=stored_request.book_id))) if locked == True: Lock.release()
def submit_job(stored_request): """Add book-request to the job queues""" redis_key1 = keys.redis_key1 lock_key1 = keys.lock_key1 q = redis_py.Queue(redis_key1) q_global_job = redis_py.Queue(redis_key1 + "global") redis = redis_py.Redis() redis_key3 = keys.redis_key3 book_key = "%s:%s:%s" % (redis_key3, stored_request.library, stored_request.book_id) book_request_key = book_key + ":requests" Lock = redis_py.Lock(lock_key1) locked = False if redis.exists(book_request_key): locked = Lock.acquire(timeout=60) request = dict(sno=stored_request.sno, email=stored_request.email) redis_py.sadd(book_request_key, json.dumps(request), request_cache=True) redundant_request = models.Request.query.filter_by( md5_book=stored_request.md5_book, confirmed=1, job_submitted=1 ).first() if redundant_request == None: reset_book_progress(stored_request.library, stored_request.book_id) md5_book = hashlib.md5(stored_request.book_id + stored_request.library).hexdigest() library_url_key = book_key + ":library_url" library_url = redis_py.get(library_url_key, True) metadata_key = book_key + ":meta_data" meta_data = redis_py.get(metadata_key, True) book = models.Book( book_id=stored_request.book_id, library=stored_request.library, md5_book=stored_request.md5_book, connected_request_sno=stored_request.sno, requests=json.dumps(request), library_url=library_url, meta_data=meta_data, ) db.session.add(book) # db.session.commit() q.add(stored_request.sno) stored_request.job_submitted = 1 db.session.commit() q_global_job.add(json.dumps(dict(library=stored_request.library, book_id=stored_request.book_id))) if locked == True: Lock.release()
def reupload(book_id, email="", key=""): subtitle = "Reupload book to Internet Archive" form = ReUpload() if email in (None, "") and key not in (None, ""): return render_template("reupload.html", subtitle=subtitle, form=form) if key != hashlib.md5(str(email) + str(keys.flask_app_secret)).hexdigest() and key not in (None, ""): return render_template("reupload.html", subtitle=subtitle, form=form) form = ReUpload() if form.validate() or key not in (None, ""): book_values = re.search("(.*):(.*)", book_id) library = book_values.group(1) Id = book_values.group(2) ia_identifier_suffix = get_valid_identifier_suffix(library, Id) reset_book_progress(library, ia_identifier_suffix) redis = redis_py.Redis() redis_key3 = keys.redis_key3 book_metadata = redis_py.get(redis_key3 + ":" + book_id + ":meta_data", True) book_key = "%s:%s:%s" % (redis_key3, library, ia_identifier_suffix) metadata_key = book_key + ":meta_data" book_request_key = book_key + ":requests" redis_py.set(metadata_key, book_metadata, True) request = dict(email=form.email.data) redis_py.sadd(book_request_key, json.dumps(request), request_cache=True) book = models.Book( book_id=ia_identifier_suffix, library=library, requests=json.dumps(request), meta_data=book_metadata ) db.session.add(book) db.session.commit() redis_key1 = keys.redis_key1 q_global_job = redis_py.Queue(redis_key1 + "global") q_global_job.add(json.dumps(dict(library=library, book_id=ia_identifier_suffix))) q = redis_py.Queue(redis_key1) q.add(json.dumps(dict(library=library, Id=Id, ia_identifier_suffix=ia_identifier_suffix))) flash(error_msg(100, email=form.email.data)) return redirect(url_for("progress", book_id=library + ":" + ia_identifier_suffix)) else: return render_template("reupload.html", subtitle=subtitle, form=form)
def manual(): subtitle = "Upload using wildcards" try: form = WildcardForm(request.form) except UnboundLocalError: form = WildcardForm() return render_template("wildcard.html", subtitle=subtitle, form=form) if request.method == "POST": if form.validate(): if form.book_url.data in (None, "") and form.book_pdf_url.data in (None, ""): return render_template("wildcard.html", subtitle=subtitle, form=form) if form.link_type.data == "wildcard": book_url = form.book_url.data if form.book_url.data not in (None, "") else "" from_no = html_escape(form.from_no.data) if form.from_no.data not in (None, "") else "" to_no = html_escape(form.to_no.data) if form.to_no.data not in (None, "") else "" info_url = re.sub("\(\*\)", str(from_no), book_url) book_url = book_url + ":" + str(from_no) + "," + str(to_no) book_url = book_url + ":" + "wildcard" elif form.link_type.data == "pdf": book_url = form.book_pdf_url.data if form.book_pdf_url.data not in (None, "") else "" info_url = book_url book_url = book_url + ":" + "pdf" library = "man" Id = hashlib.md5(book_url).hexdigest() email = html_escape(form.email.data) if form.email.data not in (None, "") else "" book = bridge.fields(library, book_url, email) verification_status_no = book.verify_fields() if verification_status_no != 0: flash(error_msg(verification_status_no, book)) return render_template("wildcard.html", subtitle=subtitle, form=form) tld = extract_base_domain(book_url) book_metadata = dict( image_url="", thumbnail_url="", printType="BOOK", subtitle="", infoLink=info_url, publicDomain=True, scanner=tld, sponser=tld, title=form.title.data if form.title.data not in (None, "") else "", author=form.author.data if form.author.data not in (None, "") else "", publisher=form.publisher.data if form.publisher.data not in (None, "") else "", publishedDate=form.date.data if form.date.data not in (None, "") else "", description=form.desc.data if form.desc.data not in (None, "") else "", language=form.language.data if form.language.data not in (None, "") else "", ) ia_identifier_suffix = get_valid_identifier_suffix(library, Id) redis = redis_py.Redis() redis_key3 = keys.redis_key3 redis_key1 = keys.redis_key1 book_key = "%s:%s:%s" % (redis_key3, library, ia_identifier_suffix) metadata_key = book_key + ":meta_data" redis_py.set(metadata_key, json.dumps(book_metadata), True) book_request_key = book_key + ":requests" request_details = dict(email=email) redis_py.sadd(book_request_key, json.dumps(request_details), request_cache=True) md5_book = hashlib.md5(ia_identifier_suffix + library).hexdigest() meta_data = json.dumps(book_metadata) book = models.Book( book_id=ia_identifier_suffix, library=library, requests=json.dumps(request_details), meta_data=meta_data, md5_book=md5_book, ) db.session.add(book) db.session.commit() q_global_job = redis_py.Queue(redis_key1 + "global") q_global_job.add(json.dumps(dict(library=library, book_id=ia_identifier_suffix))) q = redis_py.Queue(redis_key1) q.add(json.dumps(dict(library=library, Id=book_url, ia_identifier_suffix=ia_identifier_suffix))) flash(error_msg(100, email=email)) return redirect(url_for("progress", book_id=library + ":" + ia_identifier_suffix)) else: return render_template("wildcard.html", subtitle=subtitle, form=form) else: return render_template("wildcard.html", subtitle=subtitle, form=form)
def manual(): subtitle = "Upload using wildcards" try: form = WildcardForm(request.form) except UnboundLocalError: form = WildcardForm() return render_template("wildcard.html", subtitle=subtitle, form=form) if request.method == 'POST': if form.validate(): if form.book_url.data in (None, "") and form.book_pdf_url.data in (None, ""): return render_template("wildcard.html", subtitle=subtitle, form=form) if form.link_type.data == 'wildcard': book_url = form.book_url.data if form.book_url.data not in ( None, "") else "" from_no = html_escape( form.from_no.data) if form.from_no.data not in (None, "") else "" to_no = html_escape( form.to_no.data) if form.to_no.data not in (None, "") else "" info_url = re.sub('\(\*\)', str(from_no), book_url) book_url = book_url + ":" + str(from_no) + "," + str(to_no) book_url = book_url + ":" + 'wildcard' elif form.link_type.data == 'pdf': book_url = form.book_pdf_url.data if form.book_pdf_url.data not in ( None, "") else "" info_url = book_url book_url = book_url + ":" + 'pdf' library = 'man' Id = hashlib.md5(book_url).hexdigest() email = html_escape( form.email.data) if form.email.data not in (None, "") else "" book = bridge.fields(library, book_url, email) verification_status_no = book.verify_fields() if verification_status_no != 0: flash(error_msg(verification_status_no, book)) return render_template("wildcard.html", subtitle=subtitle, form=form) tld = extract_base_domain(book_url) book_metadata = dict( image_url="", thumbnail_url="", printType="BOOK", subtitle="", infoLink=info_url, publicDomain=True, scanner=tld, sponser=tld, title=form.title.data if form.title.data not in (None, "") else "", author=form.author.data if form.author.data not in (None, "") else "", publisher=form.publisher.data if form.publisher.data not in (None, "") else "", publishedDate=form.date.data if form.date.data not in (None, "") else "", description=form.desc.data if form.desc.data not in (None, "") else "", language=form.language.data if form.language.data not in (None, "") else "") ia_identifier_suffix = get_valid_identifier_suffix(library, Id) redis = redis_py.Redis() redis_key3 = keys.redis_key3 redis_key1 = keys.redis_key1 book_key = "%s:%s:%s" % (redis_key3, library, ia_identifier_suffix) metadata_key = book_key + ":meta_data" redis_py.set(metadata_key, json.dumps(book_metadata), True) book_request_key = book_key + ":requests" request_details = dict(email=email) redis_py.sadd(book_request_key, json.dumps(request_details), request_cache=True) md5_book = hashlib.md5(ia_identifier_suffix + library).hexdigest() meta_data = json.dumps(book_metadata) book = models.Book(book_id=ia_identifier_suffix, library=library, requests=json.dumps(request_details), meta_data=meta_data, md5_book=md5_book) db.session.add(book) db.session.commit() q_global_job = redis_py.Queue(redis_key1 + "global") q_global_job.add( json.dumps(dict(library=library, book_id=ia_identifier_suffix))) q = redis_py.Queue(redis_key1) q.add( json.dumps( dict(library=library, Id=book_url, ia_identifier_suffix=ia_identifier_suffix))) flash(error_msg(100, email=email)) return redirect( url_for('progress', book_id=library + ":" + ia_identifier_suffix)) else: return render_template("wildcard.html", subtitle=subtitle, form=form) else: return render_template("wildcard.html", subtitle=subtitle, form=form)