class Controller(Service):
"""
An object of this class is responsible for glueing together all the other
components. It needs the following options in the `scrapy-do` section of
the configuration:
* `project-store` - a directory for all the data and metadata.
* `job-slots` - number of jobs to run in parallel
* `completed-cap` - number of completed jobs to keep while purging the old
jobs
:param config: A :class:`Config <scrapy_do.config.Config>`.
contains the following options in the `scrapy-do` section:
"""
log = Logger()
#---------------------------------------------------------------------------
def __init__(self, config):
#-----------------------------------------------------------------------
# Configuration
#-----------------------------------------------------------------------
self.log.info('Creating controller')
self.config = config
ps = config.get_string('scrapy-do', 'project-store')
ps_abs = os.path.join(os.getcwd(), ps)
self.project_store = ps if ps.startswith('/') else ps_abs
self.job_slots = config.get_int('scrapy-do', 'job-slots')
self.completed_cap = config.get_int('scrapy-do', 'completed-cap')
self.metadata_path = os.path.join(self.project_store, 'metadata.pkl')
self.schedule_path = os.path.join(self.project_store, 'schedule.db')
self.log_dir = os.path.join(self.project_store, 'log-dir')
self.spider_data_dir = os.path.join(self.project_store, 'spider-data')
self.running_jobs = {}
self.scheduled_jobs = {}
self.counter_run = 0
self.counter_success = 0
self.counter_failure = 0
self.counter_cancel = 0
self.start_time = datetime.now()
self.listeners = set()
self.mem_usage = None
self.mem_usage_ts = None
#-----------------------------------------------------------------------
# Create all the directories
#-----------------------------------------------------------------------
dirs = [self.project_store, self.log_dir, self.spider_data_dir]
for d in dirs:
try:
os.makedirs(d)
except FileExistsError:
pass
if os.path.exists(self.metadata_path):
with open(self.metadata_path, 'rb') as f:
self.projects = pickle.load(f)
else:
self.projects = {}
with open(self.metadata_path, 'wb') as f:
pickle.dump(self.projects, f)
#-----------------------------------------------------------------------
# Set the scheduler up
#-----------------------------------------------------------------------
self.schedule = Schedule(self.schedule_path)
self.scheduler = Scheduler()
for job in self.schedule.get_jobs(Status.SCHEDULED):
self.log.info('Re-scheduling: {}'.format(str(job)))
sch_job = schedule_job(self.scheduler, job.schedule)
sch_job.do(
lambda job: self.
schedule_job(job.project, job.spider, 'now', Actor.SCHEDULER,
job.description, job.payload), job)
self.scheduled_jobs[job.identifier] = sch_job
#-----------------------------------------------------------------------
# If we have any jobs marked as RUNNING in the schedule at this point,
# it means that the daemon was killed while the jobs were running. We
# mark these jobs as pending, so that they can be restarted as soon
# as possible
#-----------------------------------------------------------------------
for job in self.schedule.get_jobs(Status.RUNNING):
self.log.info('Restarting interrupted: {}'.format(str(job)))
job.status = Status.PENDING
self._update_job(job)
#-----------------------------------------------------------------------
# Set up the service
#-----------------------------------------------------------------------
self.setName('Controller')
self.scheduler_loop = LoopingCall(self.run_scheduler)
self.crawlers_loop = LoopingCall(self.run_crawlers)
self.purger_loop = LoopingCall(self.purge_completed_jobs)
self.event_loop = LoopingCall(self.dispatch_periodic_events)
#---------------------------------------------------------------------------
def startService(self):
"""
Start the twisted related functionality.
"""
self.log.info('Starting controller')
self.scheduler_loop.start(1.)
self.crawlers_loop.start(1.)
self.purger_loop.start(10.)
self.event_loop.start(1.)
#---------------------------------------------------------------------------
def stopService(self):
"""
Stop the twisted related functionality.
"""
self.log.info('Stopping controller')
self.scheduler_loop.stop()
self.crawlers_loop.stop()
self.purger_loop.stop()
self.event_loop.stop()
return self.wait_for_running_jobs(cancel=True)
#---------------------------------------------------------------------------
@inlineCallbacks
def push_project(self, data):
"""
Register a project of a given name with the zipped code passed in data.
:param data: Binary blob with a zipped project code
:return: A deferred that gets called back with a `Project` object,
or a `ValueError` or an `EnvironmentError` failure.
"""
self.log.info('Pushing new project')
#-----------------------------------------------------------------------
# Store the data in a temoporary file
#-----------------------------------------------------------------------
tmp = tempfile.mkstemp()
with open(tmp[0], 'wb') as f:
f.write(data)
#-----------------------------------------------------------------------
# Unzip to a temporary directory
#-----------------------------------------------------------------------
temp_dir = tempfile.mkdtemp()
unzip = find_executable('unzip')
if unzip is None:
raise EnvironmentError('Please install unzip')
ret_code = yield getProcessValue(unzip, args=(tmp[1], ), path=temp_dir)
if ret_code != 0:
shutil.rmtree(temp_dir)
os.remove(tmp[1])
self.log.debug('Failed to unzip data using "{}"'.format(unzip))
raise ValueError('Not a valid zip archive')
#-----------------------------------------------------------------------
# Figure out the list of spiders
#-----------------------------------------------------------------------
config_files = glob(os.path.join(temp_dir, '**/scrapy.cfg'))
if not config_files:
shutil.rmtree(temp_dir)
os.remove(tmp[1])
raise ValueError('No project found in the archive')
config = configparser.ConfigParser()
config.read(config_files[0])
try:
name = config.get('deploy', 'project')
except (configparser.NoOptionError, configparser.NoSectionError):
shutil.rmtree(temp_dir)
os.remove(tmp[1])
raise ValueError(
'Can\'t extract project name from the config file')
temp_proj_dir = os.path.join(temp_dir, name)
if not os.path.exists(temp_proj_dir):
shutil.rmtree(temp_dir)
os.remove(tmp[1])
raise ValueError(
'Project {} not found in the archive'.format(name))
scrapy = find_executable('scrapy')
if scrapy is None:
raise EnvironmentError('Please install scrapy')
ret = yield getProcessOutputAndValue(scrapy, ('list', ),
path=temp_proj_dir)
out, err, ret_code = ret
if ret_code != 0:
shutil.rmtree(temp_dir)
os.remove(tmp[1])
raise ValueError(
'Unable to get the list of spiders. Assuming there are no '
'errors in your code, this usually means missing packages '
'in your server environment.')
spiders = out.decode('utf-8').split()
shutil.rmtree(temp_dir)
#-----------------------------------------------------------------------
# Check if we have had the project registered before and if we
# have some scheduled jobs for the spiders of this project that
# are not present in the new archive
#-----------------------------------------------------------------------
if name in self.projects:
sched_jobs = self.schedule.get_scheduled_jobs(name)
sched_spiders = [job.spider for job in sched_jobs]
for spider in sched_spiders:
if spider not in spiders:
os.remove(tmp[1])
msg = 'Spider {} is going to be removed but has ' \
'scheduled jobs'
msg = msg.format(spider)
self.log.info('Failed to push project "{}": {}'.format(
name, msg))
raise ValueError(msg)
#-----------------------------------------------------------------------
# Move to the final position and store the matadata
#-----------------------------------------------------------------------
archive = os.path.join(self.project_store, name + '.zip')
shutil.move(tmp[1], archive)
prj = Project(name, archive, spiders)
self.projects[name] = prj
with open(self.metadata_path, 'wb') as f:
pickle.dump(self.projects, f)
self.log.info('Added project "{}" with spiders {}'.format(
name, prj.spiders))
self.dispatch_event(Event.PROJECT_PUSH, prj)
returnValue(prj)
#---------------------------------------------------------------------------
def get_projects(self):
"""
Get the names of all the registred projects.
"""
return list(self.projects.keys())
#---------------------------------------------------------------------------
def get_spiders(self, project_name):
"""
Get names of all the spiders in the project.
:param project_name: Name of the project
:raises ValueError: If the project name is not known
"""
if project_name not in self.projects.keys():
raise ValueError('Unknown project ' + project_name)
return self.projects[project_name].spiders
#---------------------------------------------------------------------------
def schedule_job(self,
project,
spider,
when,
actor=Actor.USER,
description='',
payload='{}'):
"""
Schedule a crawler job.
:param project: Name of the project
:param spider: Name of the spider
:param when: A scheduling spec as handled by :meth:`schedule_job
<scrapy_do.utils.schedule_job>`
:param actor: :data:`Actor <scrapy_do.schedule.Actor>` triggering the
event
:param description: Description of the job instance (optional), defaults
to empty string
:param payload: A serialized JSON object with user data, defaults to an
empty object
:return: A string identifier of a job
"""
if project not in self.projects.keys():
raise ValueError('Unknown project ' + project)
if spider not in self.projects[project].spiders:
raise ValueError('Unknown spider {}/{}'.format(project, spider))
try:
obj = json.loads(payload)
payload = json.dumps(obj, ensure_ascii=False)
except ValueError as e:
msg = str(e)
raise ValueError('Payload is not a valid JSON string: ' + msg)
job = Job(status=Status.PENDING,
actor=actor,
schedule='now',
project=project,
spider=spider,
description=description,
payload=payload)
if when != 'now':
sch_job = schedule_job(self.scheduler, when)
sch_job.do(lambda: self.schedule_job(
project, spider, 'now', Actor.SCHEDULER, description, payload))
self.scheduled_jobs[job.identifier] = sch_job
job.status = Status.SCHEDULED
job.schedule = when
self.log.info('Scheduling: {}'.format(str(job)))
self.schedule.add_job(job)
self.dispatch_event(Event.JOB_UPDATE, job)
return job.identifier
#---------------------------------------------------------------------------
def get_jobs(self, job_status):
"""
See :meth:`Schedule.get_jobs <scrapy_do.schedule.Schedule.get_jobs>`.
"""
return self.schedule.get_jobs(job_status)
#---------------------------------------------------------------------------
def get_active_jobs(self):
"""
See :meth:`Schedule.get_active_jobs
<scrapy_do.schedule.Schedule.get_active_jobs>`.
"""
return self.schedule.get_active_jobs()
#---------------------------------------------------------------------------
def get_completed_jobs(self):
"""
See :meth:`Schedule.get_completed_jobs
<scrapy_do.schedule.Schedule.get_completed_jobs>`.
"""
return self.schedule.get_completed_jobs()
#---------------------------------------------------------------------------
def get_job(self, job_id):
"""
See :meth:`Schedule.get_job <scrapy_do.schedule.Schedule.get_job>`.
"""
return self.schedule.get_job(job_id)
#---------------------------------------------------------------------------
def get_job_logs(self, job_id):
"""
Get paths to job log files.
:return: A tuple containing paths to out and error logs or `None` if
one or both don't exist
"""
path = os.path.join(self.log_dir, job_id)
logs = []
for log in ['out', 'err']:
log_path = '{}.{}'.format(path, log)
if os.path.exists(log_path):
logs.append(log_path)
else:
logs.append(None)
return tuple(logs)
#---------------------------------------------------------------------------
def run_scheduler(self):
"""
Run the `schedule.Scheduler` jobs.
"""
self.scheduler.run_pending()
#---------------------------------------------------------------------------
@inlineCallbacks
def _run_crawler(self, project, spider, job_id, payload):
#-----------------------------------------------------------------------
# Unzip to a temporary directory
#-----------------------------------------------------------------------
temp_dir = tempfile.mkdtemp()
archive = os.path.join(self.project_store, project + '.zip')
unzip = find_executable('unzip')
if unzip is None:
raise EnvironmentError('Please install unzip')
ret_code = yield getProcessValue(unzip,
args=(archive, ),
path=temp_dir)
if ret_code != 0:
shutil.rmtree(temp_dir)
msg = 'Unable to unzip with {}. Archive corrupted?'.format(unzip)
self.log.error(msg)
raise IOError('Cannot unzip the project archive')
#-----------------------------------------------------------------------
# Run the crawler
#-----------------------------------------------------------------------
temp_proj_dir = os.path.join(temp_dir, project)
env = {'SPIDER_DATA_DIR': self.spider_data_dir}
args = ['crawl', spider]
if payload != '{}':
args += ['-a', 'payload=' + payload]
process, finished = run_process('scrapy',
args,
job_id,
self.log_dir,
env=env,
path=temp_proj_dir)
#-----------------------------------------------------------------------
# Clean up
#-----------------------------------------------------------------------
def clean_up(status):
shutil.rmtree(temp_dir)
return status
finished.addBoth(clean_up)
returnValue((process, finished))
#---------------------------------------------------------------------------
def run_crawlers(self):
"""
Spawn as many crawler processe out of pending jobs as there is free
job slots.
"""
jobs = self.schedule.get_jobs(Status.PENDING)
jobs.reverse()
while len(self.running_jobs) < self.job_slots and jobs:
self.counter_run += 1
#-------------------------------------------------------------------
# Run the job
#-------------------------------------------------------------------
job = jobs.pop()
job.status = Status.RUNNING
self._update_job(job)
# Use a placeholder until the process is actually started, so that
# we do not exceed the quota due to races.
self.running_jobs[job.identifier] = None
d = self._run_crawler(job.project, job.spider, job.identifier,
job.payload)
#-------------------------------------------------------------------
# Error starting the job
#-------------------------------------------------------------------
def spawn_errback(error, job):
self.counter_failure += 1
job.status = Status.FAILED
self._update_job(job)
self.log.error('Unable to start job {}: {}'.format(
job.identifier, exc_repr(error.value)))
del self.running_jobs[job.identifier]
#-------------------------------------------------------------------
# Job started successfully
#-------------------------------------------------------------------
def spawn_callback(value, job):
# Put the process object and the finish deferred in the
# dictionary
running_job = RunningJob(value[0], value[1], datetime.now())
self.running_jobs[job.identifier] = running_job
self.log.info('Job {} started successfully'.format(
job.identifier))
#---------------------------------------------------------------
# Finish things up
#---------------------------------------------------------------
def finished_callback(exit_code):
if exit_code == 0:
self.counter_success += 1
job.status = Status.SUCCESSFUL
else:
self.counter_failure += 1
job.status = Status.FAILED
rj = self.running_jobs[job.identifier]
job.duration = (datetime.now() - rj.time_started).seconds
msg = "Job {} exited with code {}".format(
job.identifier, exit_code)
self.log.info(msg)
self._update_job(job)
del self.running_jobs[job.identifier]
return exit_code
value[1].addCallback(finished_callback)
d.addCallbacks(spawn_callback,
spawn_errback,
callbackArgs=(job, ),
errbackArgs=(job, ))
#---------------------------------------------------------------------------
@inlineCallbacks
def wait_for_starting_jobs(self):
"""
Wait until all the crawling processes in the job slots started.
:return: A deferred triggered when all the processes have started
"""
num_starting = 1 # whatever to loop at least once
while num_starting:
num_starting = 0
for k, v in self.running_jobs.items():
if v is None:
num_starting += 1
yield twisted_sleep(0.1)
#---------------------------------------------------------------------------
@inlineCallbacks
def wait_for_running_jobs(self, cancel=False):
"""
Wait for all the running jobs to finish.
:param cancel: If `True` send a `SIGTERM` signal to each of the running
crawlers
:return: A deferred triggered when all the running jobs have
finished
"""
yield self.wait_for_starting_jobs()
#-----------------------------------------------------------------------
# Send SIGTERM if requested
#-----------------------------------------------------------------------
if cancel:
for job_id in self.running_jobs:
rj = self.running_jobs[job_id]
rj.process.signalProcess('TERM')
#-----------------------------------------------------------------------
# Wait for the jobs to finish
#-----------------------------------------------------------------------
to_finish = []
for job_id in self.running_jobs:
rj = self.running_jobs[job_id]
to_finish.append(rj.finished_d)
for d in to_finish:
yield d
#---------------------------------------------------------------------------
@inlineCallbacks
def cancel_job(self, job_id):
"""
Cancel a job.
:param job_id: A string identifier of a job
:return: A deferred that is triggered when the job is cancelled
"""
job = self.schedule.get_job(job_id)
self.log.info('Canceling: {}'.format(str(job)))
#-----------------------------------------------------------------------
# Scheduled
#-----------------------------------------------------------------------
if job.status == Status.SCHEDULED:
job.status = Status.CANCELED
self._update_job(job)
self.scheduler.cancel_job(self.scheduled_jobs[job_id])
del self.scheduled_jobs[job_id]
#-----------------------------------------------------------------------
# Pending
#-----------------------------------------------------------------------
elif job.status == Status.PENDING:
job.status = Status.CANCELED
self._update_job(job)
#-----------------------------------------------------------------------
# Running
#-----------------------------------------------------------------------
elif job.status == Status.RUNNING:
while True:
if job_id not in self.running_jobs:
raise ValueError('Job {} is not active'.format(job_id))
if self.running_jobs[job_id] is None:
yield twisted_sleep(0.1) # wait until the job starts
else:
break
rj = self.running_jobs[job_id]
rj.process.signalProcess('KILL')
yield rj.finished_d
self.counter_failure -= 1
self.counter_cancel += 1
job.status = Status.CANCELED
job.duration = (datetime.now() - rj.time_started).seconds
self._update_job(job)
#-----------------------------------------------------------------------
# Not active
#-----------------------------------------------------------------------
else:
raise ValueError('Job {} is not active'.format(job_id))
#---------------------------------------------------------------------------
def purge_completed_jobs(self):
"""
Purge all the old jobs exceeding the completed cap.
"""
old_jobs = self.get_completed_jobs()[self.completed_cap:]
if len(old_jobs):
self.log.info('Purging {} old jobs'.format(len(old_jobs)))
for job in old_jobs:
self.dispatch_event(Event.JOB_REMOVE, job.identifier)
self.schedule.remove_job(job.identifier)
for log_type in ['.out', '.err']:
log_file = os.path.join(self.log_dir,
job.identifier + log_type)
if os.path.exists(log_file):
os.remove(log_file)
#---------------------------------------------------------------------------
def remove_project(self, name):
"""
Remove the project
"""
#-----------------------------------------------------------------------
# Consistency checks
#-----------------------------------------------------------------------
if name not in self.projects:
raise ValueError('No such project: "{}"'.format(name))
sched_jobs = self.schedule.get_scheduled_jobs(name)
if len(sched_jobs) != 0:
msg = 'There are {} scheduled spiders for project "{}"'.format(
len(sched_jobs), name)
self.log.info('Failed to remove project "{}": {}'.format(
name, msg))
raise ValueError(msg)
#-----------------------------------------------------------------------
# Remove the project
#-----------------------------------------------------------------------
os.remove(self.projects[name].archive)
del self.projects[name]
with open(self.metadata_path, 'wb') as f:
pickle.dump(self.projects, f)
self.log.info('Project "{}" removed'.format(name))
self.dispatch_event(Event.PROJECT_REMOVE, name)
#---------------------------------------------------------------------------
def _update_job(self, job):
self.dispatch_event(Event.JOB_UPDATE, job)
self.schedule.commit_job(job)
#---------------------------------------------------------------------------
def add_event_listener(self, listener):
"""
Add an event listener.
"""
self.listeners.add(listener)
#---------------------------------------------------------------------------
def remove_event_listener(self, listener):
"""
Remove the event listener.
"""
self.listeners.remove(listener)
#---------------------------------------------------------------------------
def dispatch_event(self, event_type, event_data):
"""
Dispatch an event to all the listeners.
"""
for listener in self.listeners:
listener(event_type, event_data)
#---------------------------------------------------------------------------
def dispatch_periodic_events(self):
"""
Dispatch periodic events if necessary.
"""
#-----------------------------------------------------------------------
# Daemon status - send the event either every minute or whenever
# the memory usage crossed a megabyte boundary
#-----------------------------------------------------------------------
mem_usage = psutil.Process(os.getpid()).memory_info().rss
mem_usage = float(mem_usage) / 1024. / 1024.
mem_usage = int(mem_usage)
now = time.time()
if self.mem_usage is None or now - self.mem_usage_ts >= 60 or \
abs(self.mem_usage - mem_usage) >= 1:
self.mem_usage = mem_usage
self.mem_usage_ts = now
self.dispatch_event(Event.DAEMON_STATUS_CHANGE, None)
class CaptivePortal:
def run(self, essid, connect):
self._schedule = Scheduler()
self._connect = connect
self._alive = True
self._timeout_job = None
self._ap = network.WLAN(network.AP_IF)
self._ap.active(True)
self._ap.config(
essid=essid) # You can't set values before calling active(...).
poller = select.poll()
addr = self._ap.ifconfig()[0]
slim_server = self._create_slim_server(poller, essid)
dns = self._create_dns(poller, addr)
_logger.info("captive portal web server and DNS started on %s", addr)
# If no timeout is given `ipoll` blocks and the for-loop goes forever.
# With a timeout the for-loop exits every time the timeout expires.
# I.e. the underlying iterable reports that it has no more elements.
while self._alive:
# Under the covers polling is done with a non-blocking ioctl call and the timeout
# (or blocking forever) is implemented with a hard loop, so there's nothing to be
# gained (e.g. reduced power consumption) by using a timeout greater than 0.
for (s, event) in poller.ipoll(0):
# If event has bits other than POLLIN or POLLOUT then print it.
if event & ~(select.POLLIN | select.POLLOUT):
self._print_select_event(event)
slim_server.pump(s, event)
dns.pump(s, event)
slim_server.pump_expire() # Expire inactive client sockets.
self._schedule.run_pending()
slim_server.shutdown(poller)
dns.shutdown(poller)
self._ap.active(False)
def _create_slim_server(self, poller, essid):
# See the captive portal notes in docs/captive-portal.md for why we redirect not-found
# URLs and why we redirect them to an absolute URL (rather than a path like "/").
# `essid` is used as the target host but any name could be used, e.g. "wifi-setup".
config = SlimConfig(not_found_url="http://{}/".format(essid))
slim_server = SlimServer(poller, config=config)
# fmt: off
slim_server.add_module(
WebRouteModule([
RegisteredRoute(HttpMethod.GET, "/api/access-points",
self._request_access_points),
RegisteredRoute(HttpMethod.POST, "/api/access-point",
self._request_access_point),
RegisteredRoute(HttpMethod.POST, "/api/alive",
self._request_alive)
]))
# fmt: on
root = self._get_relative("www")
# fmt: off
slim_server.add_module(
FileserverModule(
{
"html": "text/html",
"css": "text/css",
"js": "application/javascript",
"woff2": "font/woff2",
"ico": "image/x-icon",
"svg": "image/svg+xml"
}, root))
# fmt: on
return slim_server
# Find a file, given a path relative to the directory contain this `.py` file.
@staticmethod
def _get_relative(filename):
return join(dirname(__file__), filename)
@staticmethod
def _create_dns(poller, addr):
addr_bytes = MicroDNSSrv.ipV4StrToBytes(addr)
def resolve(name):
_logger.info("resolving %s", name)
return addr_bytes
return MicroDNSSrv(resolve, poller)
def _request_access_points(self, request):
# Tuples are of the form (SSID, BSSID, channel, RSSI, authmode, hidden).
points = [(p[0], p[3], p[4]) for p in self._ap.scan()]
request.Response.ReturnOkJSON(points)
def _request_access_point(self, request):
data = request.GetPostedURLEncodedForm()
_logger.debug("connect request data %s", data)
ssid = data.get("ssid", None)
if not ssid:
request.Response.ReturnBadRequest()
return
password = data.get("password", None)
result = self._connect(ssid, password)
if not result:
request.Response.ReturnForbidden()
else:
request.Response.ReturnOkJSON({"message": result})
def _request_alive(self, request):
data = request.GetPostedURLEncodedForm()
timeout = data.get("timeout", None)
if not timeout:
request.Response.ReturnBadRequest()
return
_logger.debug("timeout %s", timeout)
timeout = int(timeout) + self._TOLERANCE
if self._timeout_job:
self._schedule.cancel_job(self._timeout_job)
self._timeout_job = self._schedule.every(timeout).seconds.do(
self._timed_out)
request.Response.Return(self._NO_CONTENT)
# If a client specifies a keep-alive period of Xs then they must ping again within Xs plus a fixed "tolerance".
_TOLERANCE = 1
_NO_CONTENT = 204
def _timed_out(self):
_logger.info("keep-alive timeout expired.")
self._alive = False
self._timeout_job = None
return CancelJob # Tell scheduler that we want one-shot behavior.
_POLL_EVENTS = {
select.POLLIN: "IN",
select.POLLOUT: "OUT",
select.POLLHUP: "HUP",
select.POLLERR: "ERR",
}
def _print_select_event(self, event):
mask = 1
while event:
if event & 1:
_logger.info("event %s", self._POLL_EVENTS.get(mask, mask))
event >>= 1
mask <<= 1
