def __init__(self,
processor_name,
table='entity',
type=None,
chunk_size=BATCH_ITEMS_CHUNK,
sleep_seconds=BATCH_PAUSE_SECONDS,
logger=None,
run_forever=True,
logging_level='DEBUG',
item_id=None,
process_all=False,
sql_where="",
sleep_seconds_subchunk=BATCH_PAUSE_SECONDS_SUBCHUNK,
):
if logger is not None:
self.logger=logger
else:
self.logger=Logger(logging_level, "batchprocessor", fl_stdoutlog=False, logger_name="batchprocessor")
self.processor_name = processor_name
self.table = table
self.table_props = DB_TABLES.get(table, DB_TABLES['entity'])
self.table_obj=self.table_props['obj']
self.type_field=self.table_props['type_field']
self.date_field = 'modification_date'
self.chunk_size =chunk_size
self.sleep_seconds = sleep_seconds
self.sleep_seconds_subchunk=sleep_seconds_subchunk
self.last_parsed_info = self.loadInfo()
self.run_forever = run_forever and not item_id
self.item_id = item_id
self.process_all = process_all
self.sql_where = sql_where
self.join = ''
self.type=type
self.cursor = connection.cursor()
class Processor(object):
def __init__(self,
processor_name,
table='entity',
type=None,
chunk_size=BATCH_ITEMS_CHUNK,
sleep_seconds=BATCH_PAUSE_SECONDS,
logger=None,
run_forever=True,
logging_level='DEBUG',
item_id=None,
process_all=False,
sql_where="",
sleep_seconds_subchunk=BATCH_PAUSE_SECONDS_SUBCHUNK,
):
if logger is not None:
self.logger=logger
else:
self.logger=Logger(logging_level, "batchprocessor", fl_stdoutlog=False, logger_name="batchprocessor")
self.processor_name = processor_name
self.table = table
self.table_props = DB_TABLES.get(table, DB_TABLES['entity'])
self.table_obj=self.table_props['obj']
self.type_field=self.table_props['type_field']
self.date_field = 'modification_date'
self.chunk_size =chunk_size
self.sleep_seconds = sleep_seconds
self.sleep_seconds_subchunk=sleep_seconds_subchunk
self.last_parsed_info = self.loadInfo()
self.run_forever = run_forever and not item_id
self.item_id = item_id
self.process_all = process_all
self.sql_where = sql_where
self.join = ''
self.type=type
self.cursor = connection.cursor()
def buildQuery(self):
if self.item_id is None:
wheres=list()
if not self.process_all and self.last_parsed_info and self.last_parsed_info.get('last_start', None) is not None:
wheres.append("(%s>'%s')" % (self.date_field, self.last_parsed_info['last_start']))
self.logger.info('processing entities of type %s whose %s>%s' % (self.type, self.date_field, self.last_parsed_info['last_start']))
else:
self.logger.info('processing all entities of type %s' % (self.type))
if "," in self.type:
wheres.append( "t.slug IN ('%s')" % ("','".join(self.type.split(','))))
else:
wheres.append("t.slug='%s'" %(self.type))
self.query_from_where="FROM core_%s it LEFT JOIN core_%stype t ON it.%s_id=t.id WHERE %s order by %s" % (self.table, self.table, self.type_field, ' AND '.join(wheres), self.date_field)
else:
if "," in self.item_id:
self.logger.info('processing entities %s' % (self.item_id))
self.query_from_where="FROM core_%s it WHERE id IN ('%s')" % (self.table, "','".join(self.item_id.split(',')))
else:
self.logger.info('processing entity %s' % (self.item_id))
self.query_from_where='FROM core_%s it WHERE id = %s' % (self.table, self.item_id)
def saveInfo(self, info):
pickleToFile('daemon_' + self.processor_name, info)
def loadInfo(self, default=None):
try:
#my_data=pickleFromFile('daemon_' + self.processor_name)
pickleFile=open(os.path.join(settings.PICKLER_DIR, 'daemon_' + self.processor_name), 'r')
my_data=pickle.load(pickleFile)
pickleFile.close()
self.logger.debug('Info: %s' % (my_data,))
return my_data
except Exception, err:
self.logger.critical('Error loading info: %s - %s' % (Exception, err))
return default