def submit_job(stored_request):
"""Add book-request to the job queues"""
redis_key1 = keys.redis_key1
lock_key1 = keys.lock_key1
q = redis_py.Queue(redis_key1)
q_global_job = redis_py.Queue(redis_key1 + "global")
redis = redis_py.Redis()
redis_key3 = keys.redis_key3
book_key = "%s:%s:%s" % (redis_key3, stored_request.library,
stored_request.book_id)
book_request_key = book_key + ":requests"
Lock = redis_py.Lock(lock_key1)
locked = False
if redis.exists(book_request_key):
locked = Lock.acquire(timeout=60)
request = dict(sno=stored_request.sno, email=stored_request.email)
redis_py.sadd(book_request_key, json.dumps(request), request_cache=True)
redundant_request = models.Request.query.filter_by(
md5_book=stored_request.md5_book, confirmed=1,
job_submitted=1).first()
if redundant_request == None:
reset_book_progress(stored_request.library, stored_request.book_id)
md5_book = hashlib.md5(stored_request.book_id +
stored_request.library).hexdigest()
library_url_key = book_key + ":library_url"
library_url = redis_py.get(library_url_key, True)
metadata_key = book_key + ":meta_data"
meta_data = redis_py.get(metadata_key, True)
book = models.Book(book_id=stored_request.book_id,
library=stored_request.library,
md5_book=stored_request.md5_book,
connected_request_sno=stored_request.sno,
requests=json.dumps(request),
library_url=library_url,
meta_data=meta_data)
db.session.add(book)
#db.session.commit()
q.add(stored_request.sno)
stored_request.job_submitted = 1
db.session.commit()
q_global_job.add(
json.dumps(
dict(library=stored_request.library,
book_id=stored_request.book_id)))
if locked == True:
Lock.release()
def submit_job(stored_request):
"""Add book-request to the job queues"""
redis_key1 = keys.redis_key1
lock_key1 = keys.lock_key1
q = redis_py.Queue(redis_key1)
q_global_job = redis_py.Queue(redis_key1 + "global")
redis = redis_py.Redis()
redis_key3 = keys.redis_key3
book_key = "%s:%s:%s" % (redis_key3, stored_request.library, stored_request.book_id)
book_request_key = book_key + ":requests"
Lock = redis_py.Lock(lock_key1)
locked = False
if redis.exists(book_request_key):
locked = Lock.acquire(timeout=60)
request = dict(sno=stored_request.sno, email=stored_request.email)
redis_py.sadd(book_request_key, json.dumps(request), request_cache=True)
redundant_request = models.Request.query.filter_by(
md5_book=stored_request.md5_book, confirmed=1, job_submitted=1
).first()
if redundant_request == None:
reset_book_progress(stored_request.library, stored_request.book_id)
md5_book = hashlib.md5(stored_request.book_id + stored_request.library).hexdigest()
library_url_key = book_key + ":library_url"
library_url = redis_py.get(library_url_key, True)
metadata_key = book_key + ":meta_data"
meta_data = redis_py.get(metadata_key, True)
book = models.Book(
book_id=stored_request.book_id,
library=stored_request.library,
md5_book=stored_request.md5_book,
connected_request_sno=stored_request.sno,
requests=json.dumps(request),
library_url=library_url,
meta_data=meta_data,
)
db.session.add(book)
# db.session.commit()
q.add(stored_request.sno)
stored_request.job_submitted = 1
db.session.commit()
q_global_job.add(json.dumps(dict(library=stored_request.library, book_id=stored_request.book_id)))
if locked == True:
Lock.release()
def queue(number_of_entries=100):
subtitle = "Queue for web-based upload jobs"
redis_key1 = keys.redis_key1
redis_key3 = keys.redis_key3
q_global_job = redis_py.Queue(redis_key1 + "global")
redis = redis_py.Redis()
queue = q_global_job.pop(int(number_of_entries))
total_OCR_waiting = 0
total_waiting_to_run = 0
currently_running = 0
ongoing_job_identifier = redis.get(redis_key3 + ":ongoing_job_identifier")
if queue:
for index, item in enumerate(queue):
item = json.loads(item)
queue[index] = item
upload_progress = redis_py.get(
"%s:%s:%s:upload_progress" %
(redis_key3, item['library'], item['book_id']), True)
OCR_progress = redis_py.get(
"%s:%s:%s:OCR_progress" %
(redis_key3, item['library'], item['book_id']), True)
if upload_progress == '1' and OCR_progress != '1':
queue[index].update(OCR_waiting=1)
total_OCR_waiting += 1
else:
total_waiting_to_run += 1
total_waiting_to_run = total_waiting_to_run - 1 if (
index + 1) != total_OCR_waiting else total_waiting_to_run
currently_running = 1 if (index + 1) != total_OCR_waiting else 0
return render_template("queue.html",
subtitle=subtitle,
total_OCR_waiting=total_OCR_waiting,
total_waiting_to_run=total_waiting_to_run,
queue=queue,
auto_refresh=30,
present_id=ongoing_job_identifier,
number_of_entries=int(number_of_entries),
currently_running=currently_running)
def queue(number_of_entries=100):
subtitle = "Queue for web-based upload jobs"
redis_key1 = keys.redis_key1
redis_key3 = keys.redis_key3
q_global_job = redis_py.Queue(redis_key1 + "global")
redis = redis_py.Redis()
queue = q_global_job.pop(int(number_of_entries))
total_OCR_waiting = 0
total_waiting_to_run = 0
currently_running = 0
ongoing_job_identifier = redis.get(redis_key3 + ":ongoing_job_identifier")
if queue:
for index, item in enumerate(queue):
item = json.loads(item)
queue[index] = item
upload_progress = redis_py.get(
"%s:%s:%s:upload_progress" % (redis_key3, item["library"], item["book_id"]), True
)
OCR_progress = redis_py.get("%s:%s:%s:OCR_progress" % (redis_key3, item["library"], item["book_id"]), True)
if upload_progress == "1" and OCR_progress != "1":
queue[index].update(OCR_waiting=1)
total_OCR_waiting += 1
else:
total_waiting_to_run += 1
total_waiting_to_run = total_waiting_to_run - 1 if (index + 1) != total_OCR_waiting else total_waiting_to_run
currently_running = 1 if (index + 1) != total_OCR_waiting else 0
return render_template(
"queue.html",
subtitle=subtitle,
total_OCR_waiting=total_OCR_waiting,
total_waiting_to_run=total_waiting_to_run,
queue=queue,
auto_refresh=30,
present_id=ongoing_job_identifier,
number_of_entries=int(number_of_entries),
currently_running=currently_running,
)
def reupload(book_id, email="", key=""):
subtitle = "Reupload book to Internet Archive"
form = ReUpload()
if email in (None, "") and key not in (None, ""):
return render_template('reupload.html', subtitle=subtitle, form=form)
if key != hashlib.md5(str(email) + str(keys.flask_app_secret)).hexdigest(
) and key not in (None, ""):
return render_template('reupload.html', subtitle=subtitle, form=form)
form = ReUpload()
if form.validate() or key not in (None, ""):
book_values = re.search("(.*):(.*)", book_id)
library = book_values.group(1)
Id = book_values.group(2)
ia_identifier_suffix = get_valid_identifier_suffix(library, Id)
reset_book_progress(library, ia_identifier_suffix)
redis = redis_py.Redis()
redis_key3 = keys.redis_key3
book_metadata = redis_py.get(redis_key3 + ":" + book_id + ":meta_data",
True)
book_key = "%s:%s:%s" % (redis_key3, library, ia_identifier_suffix)
metadata_key = book_key + ":meta_data"
book_request_key = book_key + ":requests"
redis_py.set(metadata_key, book_metadata, True)
request = dict(email=form.email.data)
redis_py.sadd(book_request_key,
json.dumps(request),
request_cache=True)
book = models.Book(book_id=ia_identifier_suffix,
library=library,
requests=json.dumps(request),
meta_data=book_metadata)
db.session.add(book)
db.session.commit()
redis_key1 = keys.redis_key1
q_global_job = redis_py.Queue(redis_key1 + "global")
q_global_job.add(
json.dumps(dict(library=library, book_id=ia_identifier_suffix)))
q = redis_py.Queue(redis_key1)
q.add(
json.dumps(
dict(library=library,
Id=Id,
ia_identifier_suffix=ia_identifier_suffix)))
flash(error_msg(100, email=form.email.data))
return redirect(
url_for('progress', book_id=library + ":" + ia_identifier_suffix))
else:
return render_template('reupload.html', subtitle=subtitle, form=form)
def reupload(book_id, email="", key=""):
subtitle = "Reupload book to Internet Archive"
form = ReUpload()
if email in (None, "") and key not in (None, ""):
return render_template("reupload.html", subtitle=subtitle, form=form)
if key != hashlib.md5(str(email) + str(keys.flask_app_secret)).hexdigest() and key not in (None, ""):
return render_template("reupload.html", subtitle=subtitle, form=form)
form = ReUpload()
if form.validate() or key not in (None, ""):
book_values = re.search("(.*):(.*)", book_id)
library = book_values.group(1)
Id = book_values.group(2)
ia_identifier_suffix = get_valid_identifier_suffix(library, Id)
reset_book_progress(library, ia_identifier_suffix)
redis = redis_py.Redis()
redis_key3 = keys.redis_key3
book_metadata = redis_py.get(redis_key3 + ":" + book_id + ":meta_data", True)
book_key = "%s:%s:%s" % (redis_key3, library, ia_identifier_suffix)
metadata_key = book_key + ":meta_data"
book_request_key = book_key + ":requests"
redis_py.set(metadata_key, book_metadata, True)
request = dict(email=form.email.data)
redis_py.sadd(book_request_key, json.dumps(request), request_cache=True)
book = models.Book(
book_id=ia_identifier_suffix, library=library, requests=json.dumps(request), meta_data=book_metadata
)
db.session.add(book)
db.session.commit()
redis_key1 = keys.redis_key1
q_global_job = redis_py.Queue(redis_key1 + "global")
q_global_job.add(json.dumps(dict(library=library, book_id=ia_identifier_suffix)))
q = redis_py.Queue(redis_key1)
q.add(json.dumps(dict(library=library, Id=Id, ia_identifier_suffix=ia_identifier_suffix)))
flash(error_msg(100, email=form.email.data))
return redirect(url_for("progress", book_id=library + ":" + ia_identifier_suffix))
else:
return render_template("reupload.html", subtitle=subtitle, form=form)
def progress(book_id):
redis = redis_py.Redis()
redis_key3 = keys.redis_key3
redis_key1 = keys.redis_key1
q_global_job = redis_py.Queue(redis_key1 + "global")
book_key = "%s:%s" % (redis_key3, book_id)
library = book_id.split(":")[0]
metadata_key = book_key + ":meta_data"
metadata = redis_py.get(metadata_key, True)
if metadata == None:
abort(404)
metadata = json.loads(metadata)
if metadata["title"]:
if len(metadata["title"]) > 70:
subtitle = metadata["title"][:65] + ".."
else:
subtitle = metadata["title"]
progress = dict()
ia_response_key = book_key + ":ia_response"
ia_response = redis_py.get(ia_response_key, True)
ia_response = int(ia_response) if ia_response else None
progress.update(dict(ia_response=ia_response))
download_progress_key = book_key + ":download_progress"
download_progress = redis_py.get(download_progress_key, True) if ia_response == 0 or ia_response == 3 else None
download_progress = int(download_progress) if download_progress else None
progress.update(dict(download_progress=download_progress))
upload_progress_key = book_key + ":upload_progress"
upload_progress = redis_py.get(upload_progress_key, True) if ia_response == 0 or ia_response == 3 else None
upload_progress = int(upload_progress) if upload_progress else None
progress.update(dict(upload_progress=upload_progress))
ia_identifier_key = book_key + ":ia_identifier"
try:
ia_identifier = json.loads(redis_py.get(ia_identifier_key, True))
ia_link = "http://archive.org/details/%s" % ia_identifier if ia_identifier else ""
progress.update(dict(ia_link=ia_link))
progress.update(dict(ia_identifier=ia_identifier))
except ValueError:
ia_identifier = redis_py.get(ia_identifier_key, True)
ia_link = "http://archive.org/details/%s" % ia_identifier if ia_identifier else ""
progress.update(dict(ia_link=ia_link))
progress.update(dict(ia_identifier=ia_identifier))
except:
pass
ia_identifier_suffix = book_id.split(":")[-1]
OCR_progress_key = book_key + ":OCR_progress"
OCR_progress = redis_py.get(OCR_progress_key, True) if ia_response == 0 or ia_response == 3 else None
OCR_progress = int(OCR_progress) if OCR_progress else None
progress.update(dict(OCR_progress=OCR_progress))
email_progress_key = book_key + ":email_progress"
email_progress = redis_py.get(email_progress_key, True)
email_progress = int(email_progress) if email_progress else None
progress.update(dict(email_progress=email_progress))
sno_key = book_key + ":sno"
sno = redis_py.get(sno_key, True)
sno = int(sno) if sno else None
global_queue_key = json.dumps(dict(library=library, book_id=ia_identifier_suffix))
progress.update(dict(queue_index=q_global_job.index(global_queue_key)))
progress.update(
dict(percent_complete=percent_complete(ia_response, download_progress, upload_progress, OCR_progress))
)
auto_refresh = auto_refresh_time(ia_response)
return render_template(
"progress.html",
subtitle=subtitle,
book_info=metadata,
progress=progress,
auto_refresh=auto_refresh,
request=request,
book_id=book_id,
)
def progress(book_id):
redis = redis_py.Redis()
redis_key3 = keys.redis_key3
redis_key1 = keys.redis_key1
q_global_job = redis_py.Queue(redis_key1 + "global")
book_key = "%s:%s" % (redis_key3, book_id)
library = book_id.split(':')[0]
metadata_key = book_key + ":meta_data"
metadata = redis_py.get(metadata_key, True)
if metadata == None:
abort(404)
metadata = json.loads(metadata)
if metadata['title']:
if len(metadata['title']) > 70:
subtitle = metadata['title'][:65] + ".."
else:
subtitle = metadata['title']
progress = dict()
ia_response_key = book_key + ":ia_response"
ia_response = redis_py.get(ia_response_key, True)
ia_response = int(ia_response) if ia_response else None
progress.update(dict(ia_response=ia_response))
download_progress_key = book_key + ":download_progress"
download_progress = redis_py.get(
download_progress_key,
True) if ia_response == 0 or ia_response == 3 else None
download_progress = int(download_progress) if download_progress else None
progress.update(dict(download_progress=download_progress))
upload_progress_key = book_key + ":upload_progress"
upload_progress = redis_py.get(
upload_progress_key,
True) if ia_response == 0 or ia_response == 3 else None
upload_progress = int(upload_progress) if upload_progress else None
progress.update(dict(upload_progress=upload_progress))
ia_identifier_key = book_key + ":ia_identifier"
try:
ia_identifier = json.loads(redis_py.get(ia_identifier_key, True))
ia_link = "http://archive.org/details/%s" % ia_identifier if ia_identifier else ""
progress.update(dict(ia_link=ia_link))
progress.update(dict(ia_identifier=ia_identifier))
except ValueError:
ia_identifier = redis_py.get(ia_identifier_key, True)
ia_link = "http://archive.org/details/%s" % ia_identifier if ia_identifier else ""
progress.update(dict(ia_link=ia_link))
progress.update(dict(ia_identifier=ia_identifier))
except:
pass
ia_identifier_suffix = book_id.split(':')[-1]
OCR_progress_key = book_key + ":OCR_progress"
OCR_progress = redis_py.get(
OCR_progress_key,
True) if ia_response == 0 or ia_response == 3 else None
OCR_progress = int(OCR_progress) if OCR_progress else None
progress.update(dict(OCR_progress=OCR_progress))
email_progress_key = book_key + ":email_progress"
email_progress = redis_py.get(email_progress_key, True)
email_progress = int(email_progress) if email_progress else None
progress.update(dict(email_progress=email_progress))
sno_key = book_key + ":sno"
sno = redis_py.get(sno_key, True)
sno = int(sno) if sno else None
global_queue_key = json.dumps(
dict(library=library, book_id=ia_identifier_suffix))
progress.update(dict(queue_index=q_global_job.index(global_queue_key)))
progress.update(
dict(percent_complete=percent_complete(ia_response, download_progress,
upload_progress, OCR_progress)))
auto_refresh = auto_refresh_time(ia_response)
return render_template("progress.html",
subtitle=subtitle,
book_info=metadata,
progress=progress,
auto_refresh=auto_refresh,
request=request,
book_id=book_id)
