def toggleProfiling(self):
import profiler
if self.actions.profiling.isChecked():
profiler.start()
log.notice('profiling started')
else:
profiler.stop()
log.notice('profiling stopped')
mh.changeTask('Develop', 'Profile')
def toggleProfiling(self):
import profiler
if self.actions.profiling.isChecked():
profiler.start()
log.notice('profiling started')
else:
profiler.stop()
log.notice('profiling stopped')
mh.changeTask('Utilities', 'Profile')
def testProfiler(self):
'''makes sure the profiler can be started and stopped and has the right value'''
profiler.start("my name")
time.sleep(1)
d = profiler.stop()
self.assertGreaterEqual(d.elapsed,1.00)
self.assertEqual(d.name,"my name")
self.assertEqual(d.name,"my name")
def flush(self, modeltype, donotcache=False):
donotcache = donotcache or modeltype in self._donotcache
self.assertismodelclass(modeltype)
if modeltype not in self.queues:
self.logger.debug(
"Trying to flush a queue of %s that has never been filled before."
% modeltype.__name__)
return
profiler.start('flush_' + modeltype.__name__)
requireCloseSpider = False
msg = ''
success = True
self.flush_active[modeltype] = True
try:
for deps in self.get_dependencies(
modeltype
): #If we try to flush a model that is dependent on another, flush the dependenciy first.
self.flush(deps)
queue = self.queues[modeltype]
if len(queue) > 0:
chunksize = 100
queue = self.exec_callbacks('before_flush', modeltype, queue)
with db.proxy.atomic():
for idx in range(0, len(queue), chunksize):
queue_chunked = queue[idx:idx + chunksize]
data = list(
map(lambda x: (x._data), queue_chunked)
) # Extract a list of dict from our Model queue
q = modeltype.insert_many(data)
updateablefields = {}
for fieldname in modeltype._meta.fields:
field = modeltype._meta.fields[fieldname]
if not isinstance(field, PrimaryKeyField):
updateablefields[fieldname] = field
try:
sql = self.add_onduplicate_key(
q, updateablefields
) # Manually add "On duplicate key update"
db.proxy.execute_sql(sql[0], sql[1])
except Exception as e: #We have a nasty error. Dumps useful data to a file.
filename = "%s_queuedump.txt" % (
modeltype.__name__)
msg = "%s : Flushing %s data failed. Dumping queue data to %s.\nError is %s." % (
self.__class__.__name__, modeltype.__name__,
filename, str(e))
self.logger.error("%s\n %s" %
(msg, traceback.format_exc()))
self.dumpqueue(filename, queue)
success = False
requireCloseSpider = True
if success:
#Hooks
self.exec_callbacks('after_flush', modeltype, queue)
#Stats
queueindex = modeltype
if queueindex in self.queuestats:
for spider in self.queuestats[queueindex]:
if spider not in self.stats:
self.stats[spider] = {}
if modeltype not in self.stats[spider]:
self.stats[spider][modeltype] = 0
self.stats[spider][modeltype] += self.queuestats[
queueindex][spider] # consume stats for spider
self.queuestats[queueindex][
spider] = 0 # reset to 0
#cache
reloadeddata = None
if not donotcache or issubclass(modeltype,
BasePropertyOwnerModel):
self.cache.bulkwrite(queue)
reloadeddata = self.cache.reloadmodels(
queue, queue[0]._meta.primary_key
) # Retrieve primary key (autoincrement id)
#Propkey/propval
if issubclass(
modeltype, BasePropertyOwnerModel
): # Our class has a property table defined (propkey/propval)
if reloadeddata and len(reloadeddata) > 0:
for obj in reloadeddata:
obj_spider = obj._extra_data['spider']
props = obj.getproperties()
for prop in props:
self.enqueue(prop, obj_spider)
if not self.flush_active[modeltype._meta.valmodel]:
self.flush(modeltype._meta.valmodel,
donotcache) # Flush db properties
#Remove data from cache if explicitly asked not to cache. That'll save some memory
# We delete after inserting instead of simply preventing because we want BasePropertyOwnerModel
# object to successfully respect foreign key constraints with Auto Increment fields.
if donotcache:
profiler.start('dao_deleteobj')
self.cache.bulkdeleteobj(
queue
) # Delete BasePropertyOwnerModel after provals are flushed
profiler.stop('dao_deleteobj')
self.queues[modeltype] = []
self.flush_active[modeltype] = False
profiler.stop('flush_' + modeltype.__name__)
except:
self.flush_active[modeltype] = False
raise
if requireCloseSpider:
raise CloseSpider(msg)
def testStopException(self):
'''an exception should be raised if stop() is called before start()'''
with self.assertRaises(LookupError):
profiler.stop()
def testGeneratedName(self):
'''makes sure a suitable name is generated if no specific name is passed into start()'''
profiler.start()
d = profiler.stop()
self.assertEqual(d.name,self.id().rpartition('.')[-1])
def process_spider_output(self, response, result, spider):
profiler.start('shared_queue_process')
for x in self.process_result(result, spider):
yield x
profiler.stop('shared_queue_process')
