def submit_job(stored_request): """Add book-request to the job queues""" redis_key1 = keys.redis_key1 lock_key1 = keys.lock_key1 q = redis_py.Queue(redis_key1) q_global_job = redis_py.Queue(redis_key1 + "global") redis = redis_py.Redis() redis_key3 = keys.redis_key3 book_key = "%s:%s:%s" % (redis_key3, stored_request.library, stored_request.book_id) book_request_key = book_key + ":requests" Lock = redis_py.Lock(lock_key1) locked = False if redis.exists(book_request_key): locked = Lock.acquire(timeout=60) request = dict(sno=stored_request.sno, email=stored_request.email) redis_py.sadd(book_request_key, json.dumps(request), request_cache=True) redundant_request = models.Request.query.filter_by( md5_book=stored_request.md5_book, confirmed=1, job_submitted=1).first() if redundant_request == None: reset_book_progress(stored_request.library, stored_request.book_id) md5_book = hashlib.md5(stored_request.book_id + stored_request.library).hexdigest() library_url_key = book_key + ":library_url" library_url = redis_py.get(library_url_key, True) metadata_key = book_key + ":meta_data" meta_data = redis_py.get(metadata_key, True) book = models.Book(book_id=stored_request.book_id, library=stored_request.library, md5_book=stored_request.md5_book, connected_request_sno=stored_request.sno, requests=json.dumps(request), library_url=library_url, meta_data=meta_data) db.session.add(book) #db.session.commit() q.add(stored_request.sno) stored_request.job_submitted = 1 db.session.commit() q_global_job.add( json.dumps( dict(library=stored_request.library, book_id=stored_request.book_id))) if locked == True: Lock.release()
def submit_job(stored_request): """Add book-request to the job queues""" redis_key1 = keys.redis_key1 lock_key1 = keys.lock_key1 q = redis_py.Queue(redis_key1) q_global_job = redis_py.Queue(redis_key1 + "global") redis = redis_py.Redis() redis_key3 = keys.redis_key3 book_key = "%s:%s:%s" % (redis_key3, stored_request.library, stored_request.book_id) book_request_key = book_key + ":requests" Lock = redis_py.Lock(lock_key1) locked = False if redis.exists(book_request_key): locked = Lock.acquire(timeout=60) request = dict(sno=stored_request.sno, email=stored_request.email) redis_py.sadd(book_request_key, json.dumps(request), request_cache=True) redundant_request = models.Request.query.filter_by( md5_book=stored_request.md5_book, confirmed=1, job_submitted=1 ).first() if redundant_request == None: reset_book_progress(stored_request.library, stored_request.book_id) md5_book = hashlib.md5(stored_request.book_id + stored_request.library).hexdigest() library_url_key = book_key + ":library_url" library_url = redis_py.get(library_url_key, True) metadata_key = book_key + ":meta_data" meta_data = redis_py.get(metadata_key, True) book = models.Book( book_id=stored_request.book_id, library=stored_request.library, md5_book=stored_request.md5_book, connected_request_sno=stored_request.sno, requests=json.dumps(request), library_url=library_url, meta_data=meta_data, ) db.session.add(book) # db.session.commit() q.add(stored_request.sno) stored_request.job_submitted = 1 db.session.commit() q_global_job.add(json.dumps(dict(library=stored_request.library, book_id=stored_request.book_id))) if locked == True: Lock.release()
def queue(number_of_entries=100): subtitle = "Queue for web-based upload jobs" redis_key1 = keys.redis_key1 redis_key3 = keys.redis_key3 q_global_job = redis_py.Queue(redis_key1 + "global") redis = redis_py.Redis() queue = q_global_job.pop(int(number_of_entries)) total_OCR_waiting = 0 total_waiting_to_run = 0 currently_running = 0 ongoing_job_identifier = redis.get(redis_key3 + ":ongoing_job_identifier") if queue: for index, item in enumerate(queue): item = json.loads(item) queue[index] = item upload_progress = redis_py.get( "%s:%s:%s:upload_progress" % (redis_key3, item['library'], item['book_id']), True) OCR_progress = redis_py.get( "%s:%s:%s:OCR_progress" % (redis_key3, item['library'], item['book_id']), True) if upload_progress == '1' and OCR_progress != '1': queue[index].update(OCR_waiting=1) total_OCR_waiting += 1 else: total_waiting_to_run += 1 total_waiting_to_run = total_waiting_to_run - 1 if ( index + 1) != total_OCR_waiting else total_waiting_to_run currently_running = 1 if (index + 1) != total_OCR_waiting else 0 return render_template("queue.html", subtitle=subtitle, total_OCR_waiting=total_OCR_waiting, total_waiting_to_run=total_waiting_to_run, queue=queue, auto_refresh=30, present_id=ongoing_job_identifier, number_of_entries=int(number_of_entries), currently_running=currently_running)
def queue(number_of_entries=100): subtitle = "Queue for web-based upload jobs" redis_key1 = keys.redis_key1 redis_key3 = keys.redis_key3 q_global_job = redis_py.Queue(redis_key1 + "global") redis = redis_py.Redis() queue = q_global_job.pop(int(number_of_entries)) total_OCR_waiting = 0 total_waiting_to_run = 0 currently_running = 0 ongoing_job_identifier = redis.get(redis_key3 + ":ongoing_job_identifier") if queue: for index, item in enumerate(queue): item = json.loads(item) queue[index] = item upload_progress = redis_py.get( "%s:%s:%s:upload_progress" % (redis_key3, item["library"], item["book_id"]), True ) OCR_progress = redis_py.get("%s:%s:%s:OCR_progress" % (redis_key3, item["library"], item["book_id"]), True) if upload_progress == "1" and OCR_progress != "1": queue[index].update(OCR_waiting=1) total_OCR_waiting += 1 else: total_waiting_to_run += 1 total_waiting_to_run = total_waiting_to_run - 1 if (index + 1) != total_OCR_waiting else total_waiting_to_run currently_running = 1 if (index + 1) != total_OCR_waiting else 0 return render_template( "queue.html", subtitle=subtitle, total_OCR_waiting=total_OCR_waiting, total_waiting_to_run=total_waiting_to_run, queue=queue, auto_refresh=30, present_id=ongoing_job_identifier, number_of_entries=int(number_of_entries), currently_running=currently_running, )
def reupload(book_id, email="", key=""): subtitle = "Reupload book to Internet Archive" form = ReUpload() if email in (None, "") and key not in (None, ""): return render_template('reupload.html', subtitle=subtitle, form=form) if key != hashlib.md5(str(email) + str(keys.flask_app_secret)).hexdigest( ) and key not in (None, ""): return render_template('reupload.html', subtitle=subtitle, form=form) form = ReUpload() if form.validate() or key not in (None, ""): book_values = re.search("(.*):(.*)", book_id) library = book_values.group(1) Id = book_values.group(2) ia_identifier_suffix = get_valid_identifier_suffix(library, Id) reset_book_progress(library, ia_identifier_suffix) redis = redis_py.Redis() redis_key3 = keys.redis_key3 book_metadata = redis_py.get(redis_key3 + ":" + book_id + ":meta_data", True) book_key = "%s:%s:%s" % (redis_key3, library, ia_identifier_suffix) metadata_key = book_key + ":meta_data" book_request_key = book_key + ":requests" redis_py.set(metadata_key, book_metadata, True) request = dict(email=form.email.data) redis_py.sadd(book_request_key, json.dumps(request), request_cache=True) book = models.Book(book_id=ia_identifier_suffix, library=library, requests=json.dumps(request), meta_data=book_metadata) db.session.add(book) db.session.commit() redis_key1 = keys.redis_key1 q_global_job = redis_py.Queue(redis_key1 + "global") q_global_job.add( json.dumps(dict(library=library, book_id=ia_identifier_suffix))) q = redis_py.Queue(redis_key1) q.add( json.dumps( dict(library=library, Id=Id, ia_identifier_suffix=ia_identifier_suffix))) flash(error_msg(100, email=form.email.data)) return redirect( url_for('progress', book_id=library + ":" + ia_identifier_suffix)) else: return render_template('reupload.html', subtitle=subtitle, form=form)
def reupload(book_id, email="", key=""): subtitle = "Reupload book to Internet Archive" form = ReUpload() if email in (None, "") and key not in (None, ""): return render_template("reupload.html", subtitle=subtitle, form=form) if key != hashlib.md5(str(email) + str(keys.flask_app_secret)).hexdigest() and key not in (None, ""): return render_template("reupload.html", subtitle=subtitle, form=form) form = ReUpload() if form.validate() or key not in (None, ""): book_values = re.search("(.*):(.*)", book_id) library = book_values.group(1) Id = book_values.group(2) ia_identifier_suffix = get_valid_identifier_suffix(library, Id) reset_book_progress(library, ia_identifier_suffix) redis = redis_py.Redis() redis_key3 = keys.redis_key3 book_metadata = redis_py.get(redis_key3 + ":" + book_id + ":meta_data", True) book_key = "%s:%s:%s" % (redis_key3, library, ia_identifier_suffix) metadata_key = book_key + ":meta_data" book_request_key = book_key + ":requests" redis_py.set(metadata_key, book_metadata, True) request = dict(email=form.email.data) redis_py.sadd(book_request_key, json.dumps(request), request_cache=True) book = models.Book( book_id=ia_identifier_suffix, library=library, requests=json.dumps(request), meta_data=book_metadata ) db.session.add(book) db.session.commit() redis_key1 = keys.redis_key1 q_global_job = redis_py.Queue(redis_key1 + "global") q_global_job.add(json.dumps(dict(library=library, book_id=ia_identifier_suffix))) q = redis_py.Queue(redis_key1) q.add(json.dumps(dict(library=library, Id=Id, ia_identifier_suffix=ia_identifier_suffix))) flash(error_msg(100, email=form.email.data)) return redirect(url_for("progress", book_id=library + ":" + ia_identifier_suffix)) else: return render_template("reupload.html", subtitle=subtitle, form=form)
def progress(book_id): redis = redis_py.Redis() redis_key3 = keys.redis_key3 redis_key1 = keys.redis_key1 q_global_job = redis_py.Queue(redis_key1 + "global") book_key = "%s:%s" % (redis_key3, book_id) library = book_id.split(":")[0] metadata_key = book_key + ":meta_data" metadata = redis_py.get(metadata_key, True) if metadata == None: abort(404) metadata = json.loads(metadata) if metadata["title"]: if len(metadata["title"]) > 70: subtitle = metadata["title"][:65] + ".." else: subtitle = metadata["title"] progress = dict() ia_response_key = book_key + ":ia_response" ia_response = redis_py.get(ia_response_key, True) ia_response = int(ia_response) if ia_response else None progress.update(dict(ia_response=ia_response)) download_progress_key = book_key + ":download_progress" download_progress = redis_py.get(download_progress_key, True) if ia_response == 0 or ia_response == 3 else None download_progress = int(download_progress) if download_progress else None progress.update(dict(download_progress=download_progress)) upload_progress_key = book_key + ":upload_progress" upload_progress = redis_py.get(upload_progress_key, True) if ia_response == 0 or ia_response == 3 else None upload_progress = int(upload_progress) if upload_progress else None progress.update(dict(upload_progress=upload_progress)) ia_identifier_key = book_key + ":ia_identifier" try: ia_identifier = json.loads(redis_py.get(ia_identifier_key, True)) ia_link = "http://archive.org/details/%s" % ia_identifier if ia_identifier else "" progress.update(dict(ia_link=ia_link)) progress.update(dict(ia_identifier=ia_identifier)) except ValueError: ia_identifier = redis_py.get(ia_identifier_key, True) ia_link = "http://archive.org/details/%s" % ia_identifier if ia_identifier else "" progress.update(dict(ia_link=ia_link)) progress.update(dict(ia_identifier=ia_identifier)) except: pass ia_identifier_suffix = book_id.split(":")[-1] OCR_progress_key = book_key + ":OCR_progress" OCR_progress = redis_py.get(OCR_progress_key, True) if ia_response == 0 or ia_response == 3 else None OCR_progress = int(OCR_progress) if OCR_progress else None progress.update(dict(OCR_progress=OCR_progress)) email_progress_key = book_key + ":email_progress" email_progress = redis_py.get(email_progress_key, True) email_progress = int(email_progress) if email_progress else None progress.update(dict(email_progress=email_progress)) sno_key = book_key + ":sno" sno = redis_py.get(sno_key, True) sno = int(sno) if sno else None global_queue_key = json.dumps(dict(library=library, book_id=ia_identifier_suffix)) progress.update(dict(queue_index=q_global_job.index(global_queue_key))) progress.update( dict(percent_complete=percent_complete(ia_response, download_progress, upload_progress, OCR_progress)) ) auto_refresh = auto_refresh_time(ia_response) return render_template( "progress.html", subtitle=subtitle, book_info=metadata, progress=progress, auto_refresh=auto_refresh, request=request, book_id=book_id, )
def progress(book_id): redis = redis_py.Redis() redis_key3 = keys.redis_key3 redis_key1 = keys.redis_key1 q_global_job = redis_py.Queue(redis_key1 + "global") book_key = "%s:%s" % (redis_key3, book_id) library = book_id.split(':')[0] metadata_key = book_key + ":meta_data" metadata = redis_py.get(metadata_key, True) if metadata == None: abort(404) metadata = json.loads(metadata) if metadata['title']: if len(metadata['title']) > 70: subtitle = metadata['title'][:65] + ".." else: subtitle = metadata['title'] progress = dict() ia_response_key = book_key + ":ia_response" ia_response = redis_py.get(ia_response_key, True) ia_response = int(ia_response) if ia_response else None progress.update(dict(ia_response=ia_response)) download_progress_key = book_key + ":download_progress" download_progress = redis_py.get( download_progress_key, True) if ia_response == 0 or ia_response == 3 else None download_progress = int(download_progress) if download_progress else None progress.update(dict(download_progress=download_progress)) upload_progress_key = book_key + ":upload_progress" upload_progress = redis_py.get( upload_progress_key, True) if ia_response == 0 or ia_response == 3 else None upload_progress = int(upload_progress) if upload_progress else None progress.update(dict(upload_progress=upload_progress)) ia_identifier_key = book_key + ":ia_identifier" try: ia_identifier = json.loads(redis_py.get(ia_identifier_key, True)) ia_link = "http://archive.org/details/%s" % ia_identifier if ia_identifier else "" progress.update(dict(ia_link=ia_link)) progress.update(dict(ia_identifier=ia_identifier)) except ValueError: ia_identifier = redis_py.get(ia_identifier_key, True) ia_link = "http://archive.org/details/%s" % ia_identifier if ia_identifier else "" progress.update(dict(ia_link=ia_link)) progress.update(dict(ia_identifier=ia_identifier)) except: pass ia_identifier_suffix = book_id.split(':')[-1] OCR_progress_key = book_key + ":OCR_progress" OCR_progress = redis_py.get( OCR_progress_key, True) if ia_response == 0 or ia_response == 3 else None OCR_progress = int(OCR_progress) if OCR_progress else None progress.update(dict(OCR_progress=OCR_progress)) email_progress_key = book_key + ":email_progress" email_progress = redis_py.get(email_progress_key, True) email_progress = int(email_progress) if email_progress else None progress.update(dict(email_progress=email_progress)) sno_key = book_key + ":sno" sno = redis_py.get(sno_key, True) sno = int(sno) if sno else None global_queue_key = json.dumps( dict(library=library, book_id=ia_identifier_suffix)) progress.update(dict(queue_index=q_global_job.index(global_queue_key))) progress.update( dict(percent_complete=percent_complete(ia_response, download_progress, upload_progress, OCR_progress))) auto_refresh = auto_refresh_time(ia_response) return render_template("progress.html", subtitle=subtitle, book_info=metadata, progress=progress, auto_refresh=auto_refresh, request=request, book_id=book_id)