def scan_inode_distributed(case, inode_id, scanners, cookie):
""" Schedules a job to scan the inode by an available worker.
Note this will actually happen out of process.
"""
Farm.post_job('Scan', dict(case=case, inode_id=inode_id, scanners=scanners), cookie)
return
#return scan_inode(case, inode_id, factories, cookie)
pdbh = DB.DBO()
## We require that all scanning calls have a valid cookie. The
## following helps to trap bugs by scanners not passing the cookie
## along.
if cookie == 0:
raise RuntimeError( "Invalid cookie")
## This is a cookie used to identify our requests so that we
## can check they have been done later.
pdbh.insert("jobs",
command = 'Scan',
arg1 = case,
arg2 = inode_id,
arg3 = ','.join([f.name for f in factories]),
cookie=cookie,
_fast = True
)
## This is running in the worker itself - we can not wake up the
## other workers from here.
Farm.wake_workers()
def schedule_inode_index_sql(case, sql, word_id, cookie='', unique=True):
dict_version = get_dict_version()
## If we want detailed scan we want to set bit 30 of the dict_version
if not unique:
desired_version = dict_version | 2**30
else:
desired_version = dict_version
## Now check the inode to see if its out dated
dbh = DB.DBO(case)
dbh2 = DB.DBO(case)
pdbh = DB.DBO()
dbh.execute("select inode_id from "
" inode where (inode.version < %r or "
" (inode.version & (pow(2,30) -1)) < %r ) "
" and (inode_id in (%s))" ,
dict_version, desired_version, sql)
for row in dbh:
dbh2.update('inode',
where = "inode_id = %s" % row['inode_id'],
desired_version = desired_version,
_fast=True)
pdbh.insert('jobs',
cookie = cookie,
command = 'Index',
_fast = True,
arg1 = case,
arg2 = row['inode_id'],
arg3 = desired_version) # The desired version we
# want to be in
## Wake the workers:
Farm.wake_workers()
def wait_for_scan(self, cookie):
""" Waits for scanners to complete """
pdbh = DB.DBO()
## Often this process owns a worker as well. In that case we can wake it up:
import pyflag.Farm as Farm
Farm.wake_workers()
## Wait until there are no more jobs left.
while 1:
pdbh.execute("select count(*) as total from jobs where cookie=%r and arg1=%r", (cookie,
self.environment._CASE))
row = pdbh.fetch()
if row and row['total']==0: break
time.sleep(1)
def wait_for_scan(self, cookie):
""" Waits for scanners to complete """
pdbh = DB.DBO()
## Often this process owns a worker as well. In that case we can wake it up:
import pyflag.Farm as Farm
Farm.wake_workers()
## Wait until there are no more jobs left.
while 1:
pdbh.execute(
"select count(*) as total from jobs where cookie=%r and arg1=%r",
(cookie, self.environment._CASE))
row = pdbh.fetch()
if row and row['total'] == 0: break
time.sleep(1)
def schedule_index(case, inode_ids, word, word_type, unique=True):
""" This function schedules an indexing job on the inode specified with the word and index type specified.
We check to see if the index is already in the dictionary and if
said inode is up to the relevant dictionary version.
unique requests the indexer to produce a single hit for each inode
(i.e. it only shows unique hits). If unique is False the indexer
will generate offsets for all hits in this inode.
"""
word_id = insert_dictionary_word(word, word_type)
if type(inode_ids)!=type(list):
inode_ids = (inode_ids,)
for inode_id in inode_ids:
schedule_inode_index(case, inode_id, word_id)
## Wake the workers:
Farm.wake_workers()
said inode is up to the relevant dictionary version.
unique requests the indexer to produce a single hit for each inode
(i.e. it only shows unique hits). If unique is False the indexer
will generate offsets for all hits in this inode.
"""
word_id = insert_dictionary_word(word, word_type)
if type(inode_ids) != type(list):
inode_ids = (inode_ids, )
for inode_id in inode_ids:
schedule_inode_index(case, inode_id, word_id)
## Wake the workers:
Farm.wake_workers()
def count_outdated_inodes(case, sql, word_id=None, unique=True):
""" This function counts the number of inodes outstanding to be scanned.
sql is a subquery which is expected to return a list of inode_ids.
word_id is the word which we count the outdated inodes with
respect to. If word_id is None, or omitted we simply count all
inodes in the sql provided.
unique is the type of index (a unique index of a detailed index).
"""
dbh = DB.DBO(case)
