def __init__(self, name):
self.name = name
self.cwd = os.getcwd()
self.on_start_item = Event()
self.on_complete_item = Event()
self.on_fail_item = Event()
self.on_finish_item = Event()
def __init__(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
self.on_output = Event()
self.on_end = Event()
self.pipe = None
def __init__(self, pipeline, item_id, item_number,
keep_data=False, prepare_data_directory=True,
**kwargs):
super(Item, self).__init__(**kwargs)
self._pipeline = pipeline
self._item_id = item_id
self._item_number = item_number
self._keep_data = keep_data
self.may_be_canceled = False
self._item_state = self.ItemState.running
self._task_status = {}
self._start_time = time.time()
self._end_time = None
self._errors = []
self._last_output = ""
self.on_output = Event()
self.on_error = Event()
self.on_item_state = Event()
self.on_task_status = Event()
# Legacy events
self.on_cancel = Event()
self.on_complete = Event()
self.on_fail = Event()
self.on_finish = Event()
self.on_item_state.handle(self._dispatch_legacy_events)
if prepare_data_directory:
self.prepare_data_directory()
def __init__(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
self.ioloop = None
self.master_fd = None
self.master = None
self.pipe = None
self.stdin = None
self.on_output = Event()
self.on_end = Event()
def __init__(self, *tasks):
self.cwd = os.getcwd()
self.on_start_item = Event()
self.on_complete_item = Event()
self.on_fail_item = Event()
self.on_cancel_item = Event()
self.on_finish_item = Event()
self.items_in_pipeline = set()
self.tasks = []
for task in tasks:
self.add_task(task)
def __init__(self,
pipeline,
item_id,
item_number,
properties=None,
keep_data=False):
self.pipeline = pipeline
self.item_id = item_id
self.item_number = item_number
self.properties = properties or {}
self.keep_data = keep_data
self.may_be_canceled = False
self.canceled = False
self.completed = False
self.failed = False
self._errors = []
self._last_output = ""
self.task_status = {}
self.on_output = Event()
self.on_error = Event()
self.on_task_status = Event()
self.on_property = Event()
self.on_cancel = Event()
self.on_complete = Event()
self.on_fail = Event()
self.on_finish = Event()
self.prepare_data_directory()
def __init__(self, stop_file=None, concurrent_items=1, max_items=None, keep_data=False):
self.pipeline = None
self.concurrent_items = concurrent_items
self.max_items = max_items
self.keep_data = keep_data
self.item_count = 0
self.active_items = set()
self.stop_flag = False
self.stop_file = stop_file
self.initial_stop_file_mtime = self.stop_file_mtime()
self.on_status = Event()
self.on_create_item = Event()
self.on_pipeline_start_item = Event()
self.on_pipeline_finish_item = Event()
self.on_finish = Event()
if stop_file:
ioloop.PeriodicCallback(self.check_stop_file, 5000).start()
def __init__(self, *tasks):
self.cwd = os.getcwd()
self.data_dir = os.path.join(self.cwd, "data")
self.on_start_item = Event()
self.on_complete_item = Event()
self.on_fail_item = Event()
self.on_cancel_item = Event()
self.on_finish_item = Event()
self.on_cleanup = Event()
self.on_stop_requested = Event()
self.on_stop_canceled = Event()
self.project = None
self.items_in_pipeline = set()
self.tasks = []
for task in tasks:
self.add_task(task)
def __init__(self,
projects_dir,
data_dir,
warrior_hq_url,
real_shutdown=False,
keep_data=False):
if not os.access(projects_dir, os.W_OK):
raise Exception("Couldn't write to projects directory: %s" %
projects_dir)
if not os.access(data_dir, os.W_OK):
raise Exception("Couldn't write to data directory: %s" % data_dir)
self.projects_dir = projects_dir
self.data_dir = data_dir
self.warrior_hq_url = warrior_hq_url
self.real_shutdown = real_shutdown
self.keep_data = keep_data
# disable the password prompts
self.gitenv = dict(
list(os.environ.items()) + list({
'GIT_ASKPASS': '******',
'SSH_ASKPASS': '******'
}.items()))
self.warrior_id = StringConfigValue(
name="warrior_id",
title="Warrior ID",
description="The unique number of your warrior instance.",
editable=False)
self.selected_project_config_value = StringConfigValue(
name="selected_project",
title="Selected project",
description="The project (to be continued when the warrior "
"restarts).",
default="none",
editable=False)
self.downloader = StringConfigValue(
name="downloader",
title="Your nickname",
description="We use your nickname to show your results on our "
"tracker. Letters and numbers only.",
regex="^[-_a-zA-Z0-9]{3,30}$",
advanced=False)
self.concurrent_items = NumberConfigValue(
name="concurrent_items",
title="Concurrent items",
description="How many items should the warrior download at a "
"time? (Max: 6)",
min=1,
max=6,
default=2)
self.http_username = StringConfigValue(
name="http_username",
title="HTTP username",
description="Enter a username to protect the web interface, "
"or leave empty.",
default="")
self.http_password = StringConfigValue(
name="http_password",
title="HTTP password",
description="Enter a password to protect the web interface, "
"or leave empty.",
default="")
self.config_manager = ConfigManager(
os.path.join(projects_dir, "config.json"))
self.config_manager.add(self.warrior_id)
self.config_manager.add(self.selected_project_config_value)
self.config_manager.add(self.downloader)
self.config_manager.add(self.concurrent_items)
self.config_manager.add(self.http_username)
self.config_manager.add(self.http_password)
self.bandwidth_monitor = BandwidthMonitor("eth0")
self.bandwidth_monitor.update()
self.runner = Runner(concurrent_items=self.concurrent_items,
keep_data=self.keep_data)
self.runner.on_finish += self.handle_runner_finish
self.current_project_name = None
self.current_project = None
self.selected_project = None
self.projects = {}
self.installed_projects = set()
self.failed_projects = set()
self.on_projects_loaded = Event()
self.on_project_installing = Event()
self.on_project_installed = Event()
self.on_project_installation_failed = Event()
self.on_project_refresh = Event()
self.on_project_selected = Event()
self.on_status = Event()
self.on_broadcast_message_received = Event()
self.http_client = AsyncHTTPClient()
self.installing = False
self.shut_down_flag = False
self.reboot_flag = False
io_loop = ioloop.IOLoop.instance()
def update_warror_callback():
io_loop.add_future(self.update_warrior_hq(),
lambda fut: fut.result())
def update_project_callback():
io_loop.add_future(self.update_project(), lambda fut: fut.result())
self.hq_updater = ioloop.PeriodicCallback(update_warror_callback,
10 * 60 * 1000)
self.project_updater = ioloop.PeriodicCallback(update_project_callback,
30 * 60 * 1000)
self.forced_reboot_timeout = None
self.lat_lng = None
self.find_lat_lng()
self.install_output = None
self.broadcast_message = None
self.contacting_hq_failed = False
class Item(ItemData):
'''A thing, or work unit, that needs to be downloaded.
It has properties that are filled by the :class:`Task`.
An Item behaves like a mutable mapping.
.. note::
State belonging to a item should be stored on the actual item
itself. That is, do not store variables onto a :class:`Task` unless
you know what you are doing.
'''
class ItemState(object):
'''State of the item.'''
running = "running"
canceled = "canceled"
completed = "completed"
failed = "failed"
class TaskStatus(object):
'''Status of happened on a task.'''
running = "running"
completed = "completed"
failed = "failed"
def __init__(self, pipeline, item_id, item_number,
keep_data=False, prepare_data_directory=True,
**kwargs):
super(Item, self).__init__(**kwargs)
self._pipeline = pipeline
self._item_id = item_id
self._item_number = item_number
self._keep_data = keep_data
self.may_be_canceled = False
self._item_state = self.ItemState.running
self._task_status = {}
self._start_time = time.time()
self._end_time = None
self._errors = []
self._last_output = ""
self.on_output = Event()
self.on_error = Event()
self.on_item_state = Event()
self.on_task_status = Event()
# Legacy events
self.on_cancel = Event()
self.on_complete = Event()
self.on_fail = Event()
self.on_finish = Event()
self.on_item_state.handle(self._dispatch_legacy_events)
if prepare_data_directory:
self.prepare_data_directory()
def __hash__(self):
return hash(self._item_id)
@property
def item_state(self):
return self._item_state
@property
def task_status(self):
return self._task_status
@property
def start_time(self):
return self._start_time
@property
def end_time(self):
return self._end_time
@property
def canceled(self):
return self._item_state == self.ItemState.canceled
@property
def completed(self):
return self._item_state == self.ItemState.completed
@property
def failed(self):
return self._item_state == self.ItemState.failed
@property
def finished(self):
return self.canceled or self.completed or self.failed
@property
def pipeline(self):
return self._pipeline
@property
def item_id(self):
return self._item_id
@property
def item_number(self):
return self._item_number
def prepare_data_directory(self):
dirname = os.path.join(self._pipeline.data_dir, self._item_id)
self["data_dir"] = dirname
if os.path.isdir(dirname):
shutil.rmtree(dirname)
os.makedirs(dirname)
def clear_data_directory(self):
if not self._keep_data:
dirname = self["data_dir"]
if os.path.isdir(dirname):
shutil.rmtree(dirname)
def log_output(self, data, full_line=True):
if isinstance(data, seesaw.six.binary_type):
try:
data = data.decode('utf8', 'replace')
except UnicodeError:
data = data.decode('ascii', 'replace')
if full_line and len(data) > 0:
if data[0] != "\n" and len(self._last_output) > 0 and \
self._last_output[-1] != "\n":
data = "\n" + data
if data[-1] != "\n":
data += "\n"
self._last_output = data
self.on_output(self, data)
def log_error(self, task, *args):
self._errors.append((task, args))
self.on_error(self, task, *args)
def set_task_status(self, task, status):
if task in self._task_status:
old_status = self._task_status[task]
else:
old_status = None
if status != old_status:
self._task_status[task] = status
self.on_task_status(self, task, status, old_status)
def cancel(self):
assert not self.canceled
self.clear_data_directory()
self._item_state = self.ItemState.canceled
self._end_time = time.time()
self.on_item_state(self, self._item_state)
def complete(self):
assert not self.completed
self.clear_data_directory()
self._item_state = self.ItemState.completed
self._end_time = time.time()
self.on_item_state(self, self._item_state)
def fail(self):
assert not self.failed
self.clear_data_directory()
self._item_state = self.ItemState.failed
self._end_time = time.time()
self.on_item_state(self, self._item_state)
def _dispatch_legacy_events(self, item, state):
if state == self.ItemState.failed:
self.on_fail(item)
elif state == self.ItemState.completed:
self.on_complete(item)
elif state == self.ItemState.canceled:
self.on_cancel(item)
else:
raise Exception('Unknown event')
self.on_finish(item)
def description(self):
return "Item %s" % self.properties.get("item_name", "")
def __str__(self):
return "<Item '{0}' {1} {2}>".format(
self.properties.get("item_name", ""),
self._item_id, self._item_state
)
def __init__(self, properties=None):
super(ItemData, self).__init__()
self._properties = properties or {}
self.on_property = Event()