def ping_db(db):
"""Checks and connects, if database not connected"""
try:
command = "select 1;"
db.execute(command)
except:
db = mysql_py.Db()
return db
def remove_request_db(users_request, book_key):
"""Remove database entry of the users request"""
db = mysql_py.Db()
params = book_key.split(":")
book_id = params[2]
library = params[1]
for u in users_request:
command = "delete from global_request where data = %s and book_id = %s and library = %s;"
db.execute(command, u, book_id, library)
def remove_from_db(users_request):
"""Remove database entry of the request."""
db = mysql_py.Db()
for u in users_request:
info = json.loads(u)
if 'sno' in info.keys():
sno = int(info['sno'])
command = "delete from request where sno = %s;"
db.execute(command, sno)
db.close()
def manager(q_mass_worker):
db = mysql_py.Db()
while True:
book_id = False
while book_id == False:
book_id = q_mass_worker.pop_and_remove(wait_type='nonblocking')
if book_id == False:
book_id = get_id_from_another_worker(mass_worker_key)
if book_id == False:
time.sleep(1)
try:
book_id = json.loads(book_id)
except ValueError:
pass
if isinstance(book_id, list):
email = book_id[0]
no_of_uploads = book_id[1]
if email not in (None, ""):
send_email(email, no_of_uploads)
continue
ia_w = IaWorker(book_id)
stored_identifier = ia_w.stored_copy_check()
if stored_identifier != None:
continue
db = ping_db(db)
md5_book = hashlib.md5(ia_w.Id + ia_w.library).hexdigest()
redundancy_book = db.execute(
"select count(*) from request where md5_book=%s and confirmed=1 and job_submitted=1;",
md5_book)
if redundancy_book[0][0] != 0:
continue
metadata_status = ia_w.set_metadata()
if isinstance(metadata_status, (int, long, float, complex)):
if metadata_status == 7:
log.write(
'%s %s API limit exceeded Sleeping with book_id:%s\n' %
(datetime.now(), __worker_name, book_id))
log.flush()
time.sleep(seconds_until_google_quota_refresh())
q_mass_worker.add(book_id)
continue
elif metadata_status == 3:
log.write("%s %s Book download restricted\n" %
(datetime.now(), book_id))
log.flush()
continue
elif metadata_status == 2:
log.write("%s %s Not Public Domain\n" %
(datetime.now(), book_id))
log.flush()
continue
elif metadata_status == 0:
pass
else:
log.write(
"%s Metadata Error, library:%s, ID:%s, status:%s\n" %
(datetime.now(), 'gb', book_id, metadata_status))
log.flush()
continue
"""
ia_identifier_found = ia_w.check_in_IA(ia_w.library, ia_w.Id)
if ia_identifier_found is not False:
ia_w.save_ia_identifier(ia_identifier_found)
continue
"""
if not os.path.isfile(ia_w.pdf_path):
download_status = gb.download_book(ia_w.Id)
if download_status != 0:
log.write("%s Download Error, library:%s, ID:%s\n" %
(datetime.now(), 'gb', book_id))
log.flush()
continue
download_progress_key = ia_w.book_key + ":download_progress"
redis_py.set(download_progress_key, 1, True)
try:
upload_status = ia_w.upload_to_IA(ia_w.library, ia_w.Id)
if str(upload_status) == "[<Response [200]>]":
upload_progress_key = ia_w.book_key + ":upload_progress"
redis_py.set(upload_progress_key, 1, True)
ia_w.save_ia_identifier(ia_w.ia_identifier)
except:
filename = ia_w.filename
command = "rm %s" % (filename)
try:
subprocess.check_call(command, shell=True)
except:
log.write("%s Command rm %s failed" %
(datetime.now(), filename))
log.flush()
def __init__(self, value):
"""Assign variable, and get metadata from cache"""
redis_key3 = keys.redis_key3
self.redis_key3 = redis_key3
self.redis = redis_py.Redis()
if isinstance(value, (int, long, float, complex)):
db = mysql_py.Db()
values = db.execute(
'select library, book_id from request where sno = %s;',
value)[0]
db.close()
self.library = values[0]
self.Id = values[1].encode('utf-8')
self.book_key = "%s:%s:%s" % (redis_key3, self.library, self.Id)
self.redis.set(redis_key3 + ":ongoing_job_identifier", self.Id)
self.ia_identifier = None
self.id_for_key = self.Id
else:
self.library = value['library']
self.Id = value['Id']
self.ia_identifier = "bub_" + self.library + "_" + value[
'ia_identifier_suffix']
self.book_key = "%s:%s:%s" % (redis_key3, self.library,
value['ia_identifier_suffix'])
self.redis.set(redis_key3 + ":ongoing_job_identifier",
value['ia_identifier_suffix'])
self.id_for_key = value['ia_identifier_suffix']
if '/' not in self.id_for_key:
self.redis_output_file_key = "%s:%s:%s:output_file" % (
redis_key3, self.library, self.id_for_key)
else:
self.redis_output_file_key = "%s:%s:%s:output_file" % (
redis_key3, self.library, hashlib.md5(
self.id_for_key).hexdigest())
self.library_name = bridge.lib_module(self.library)[1]
metadata_key = self.book_key + ":meta_data"
metadata = redis_py.get(metadata_key, True)
info = json.loads(metadata)
try:
self.title = info['title'].encode(
"utf-8") + " " + info['subtitle'].encode("utf-8")
except:
self.title = str(info['title'].encode("utf-8")) + " " + str(
info['subtitle'])
self.author = info['author'].encode("utf-8")
self.publisher = info['publisher'].encode("utf-8")
self.description = info['description'].replace("\n",
"").encode("utf-8")
self.printType = info['printType'].encode("utf-8")
self.publishedDate = re.sub("[^0123456789/.-]", "",
info['publishedDate'].encode("utf-8"))
self.infoLink = info['infoLink']
self.publicDomain = info['publicDomain']
language_code = info['language'].encode("utf-8")
if self.publishedDate not in (None, ""):
try:
self.publishedDate = re.sub('[x?]', '0', self.publishedDate)
self.year = parser.parse(self.publishedDate).year
self.month = parser.parse(self.publishedDate).month
self.day = parser.parse(self.publishedDate).day
except:
self.year = ""
self.month = ""
self.day = ""
else:
self.year = ""
self.month = ""
self.day = ""
try:
self.language = lang_code(language_code)
except:
self.language = ""
self.pdf_path = "/data/scratch/BUB_downloads/bub_%s_%s.pdf" % (
self.library, self.Id)
self.scanner = info['scanner']
self.sponser = info['sponser']
# Maintained at https://github.com/rohit-dua/BUB
import redis
import time
import mysql_py
import keys
def Redis():
redis_host = keys.redis_host
redis_port = keys.redis_port
return redis.Redis(host=redis_host, port=redis_port)
db = mysql_py.Db()
redisg = Redis()
class Lock(object):
"""A Lock based on Redis Key value system."""
def __init__(self, key):
self.key = key
self.redis = Redis()
def acquire(self, timeout=0):
while self.redis.get(self.key) is '1':
time.sleep(0.1)
continue
self.redis.set(self.key, '1')
if timeout != 0: