def _loop(self, args, opts): if settings.get('MEMDEBUG_WITH_GUPPY', False) and guppy: heapy = guppy.hpy() task = Task().next(locked=0, completed=0) if task: task.lock() cmd = ['python', os.path.join(os.getcwd(), 'scrapy-ctl.py'), 'run'] cmd.append('--task-id=%s'%task.id) if opts.child_logfile: cmd.append('--logfile=%s'%opts.child_logfile) cmd.append('--child') task.start = datetime.now() process = subprocess.Popen(cmd, shell=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True) task.result, task.errors = process.communicate() task.finish = datetime.now() task.completed = 1 task.save() timetext.LANG = 'en' total = task.finish - task.start log.msg('Finished: %s(%s) in %s'%(task.name, task.id, timetext.stringify(total)), level=log.INFO, domain=task.domain) if settings.get('MEMDEBUG_WITH_GUPPY', False) and guppy: log.msg(heapy.heap(), level=log.DEBUG) heapy.setref() else: time.sleep(30)