def recordPID(self):
"""Save the pid of the AppServer to a file."""
if self.setting('PidFile') is None:
self._pidFile = None
return
pidpath = self.serverSidePath(self.setting('PidFile'))
try:
self._pidFile = PidFile(pidpath)
except ProcessRunning:
raise ProcessRunning('The file ' + pidpath + ' exists\n'
'and contains a process id corresponding to a running process.\n'
'This indicates that there is an AppServer already running.\n'
'If this is not the case, delete this file and restart the AppServer.')
def stop(*args, **kw):
"""Stop the AppServer (which may be in a different process)."""
print "Stopping the AppServer..."
workDir = kw.get('workDir')
if workDir:
pidfile = None
else:
if globalAppServer:
pidfile = globalAppServer._pidFile
else:
pidfile = None
if not pidfile:
workDir = os.path.dirname(__file__)
if not pidfile:
pidfile = PidFile(os.path.join(workDir, 'appserver.pid'), create=0)
try:
pidfile.kill()
except Exception:
from traceback import print_exc
print_exc(1)
print "WebKit cannot terminate the running process."
help=f'output (default: {CHOICE_OUTPUT[0]})',
type=str,
choices=CHOICE_OUTPUT,
default=CHOICE_OUTPUT[0])
arg_s = parser.add_argument(
'-s',
'--silent',
help='turn on silent mode, use with output=stdout (default: False)',
action='store_true')
arg = vars(parser.parse_args())
if arg['silent'] and arg['output'] != 'stdout':
msg = "isn't used with argument -o|--output equal 'stdout'"
raise argparse.ArgumentError(arg_s, msg)
arguments_for_crawler = functools.reduce(
lambda x, y: x + y, [f"{key}={value} " for key, value in arg.items()],
"")
logging.info(f"Crawler starts with options: {arguments_for_crawler}")
# call crawler with given parameters
# command for running looks like: scrapy runspider spider.py -a [arg1=val1
# arg2=val2 ...]
# call(["scrapy", "runspider", os.path.join("crawler", "WikiSpider.py"),
# "-a", f'arg={arguments_for_crawler}'])
with PidFile():
logging.info("Start crawler")
process.crawl(WikiSpider, arg=arguments_for_crawler)
process.start(
) # the script will block here until the crawling is finished
