コード例 #1
0
ファイル: task.py プロジェクト: Vukky123/seesaw-kit
 def __init__(self, name):
     self.name = name
     self.cwd = os.getcwd()
     self.on_start_item = Event()
     self.on_complete_item = Event()
     self.on_fail_item = Event()
     self.on_finish_item = Event()
コード例 #2
0
    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs

        self.on_output = Event()
        self.on_end = Event()

        self.pipe = None
コード例 #3
0
    def __init__(self, pipeline, item_id, item_number,
                 keep_data=False, prepare_data_directory=True,
                 **kwargs):
        super(Item, self).__init__(**kwargs)

        self._pipeline = pipeline
        self._item_id = item_id
        self._item_number = item_number
        self._keep_data = keep_data
        self.may_be_canceled = False

        self._item_state = self.ItemState.running
        self._task_status = {}
        self._start_time = time.time()
        self._end_time = None
        self._errors = []
        self._last_output = ""

        self.on_output = Event()
        self.on_error = Event()
        self.on_item_state = Event()
        self.on_task_status = Event()

        # Legacy events
        self.on_cancel = Event()
        self.on_complete = Event()
        self.on_fail = Event()
        self.on_finish = Event()
        self.on_item_state.handle(self._dispatch_legacy_events)

        if prepare_data_directory:
            self.prepare_data_directory()
コード例 #4
0
    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs
        self.ioloop = None
        self.master_fd = None
        self.master = None
        self.pipe = None
        self.stdin = None

        self.on_output = Event()
        self.on_end = Event()
コード例 #5
0
ファイル: pipeline.py プロジェクト: pombredanne/patch-grab
  def __init__(self, *tasks):
    self.cwd = os.getcwd()
    self.on_start_item = Event()
    self.on_complete_item = Event()
    self.on_fail_item = Event()
    self.on_cancel_item = Event()
    self.on_finish_item = Event()

    self.items_in_pipeline = set()
    self.tasks = []
    for task in tasks:
      self.add_task(task)
コード例 #6
0
ファイル: item.py プロジェクト: pombredanne/patch-grab
    def __init__(self,
                 pipeline,
                 item_id,
                 item_number,
                 properties=None,
                 keep_data=False):
        self.pipeline = pipeline
        self.item_id = item_id
        self.item_number = item_number
        self.properties = properties or {}
        self.keep_data = keep_data

        self.may_be_canceled = False
        self.canceled = False
        self.completed = False
        self.failed = False
        self._errors = []
        self._last_output = ""
        self.task_status = {}

        self.on_output = Event()
        self.on_error = Event()
        self.on_task_status = Event()
        self.on_property = Event()
        self.on_cancel = Event()
        self.on_complete = Event()
        self.on_fail = Event()
        self.on_finish = Event()

        self.prepare_data_directory()
コード例 #7
0
  def __init__(self, stop_file=None, concurrent_items=1, max_items=None, keep_data=False):
    self.pipeline = None
    self.concurrent_items = concurrent_items
    self.max_items = max_items
    self.keep_data = keep_data

    self.item_count = 0
    self.active_items = set()
    self.stop_flag = False
    self.stop_file = stop_file
    self.initial_stop_file_mtime = self.stop_file_mtime()

    self.on_status = Event()
    self.on_create_item = Event()
    self.on_pipeline_start_item = Event()
    self.on_pipeline_finish_item = Event()
    self.on_finish = Event()

    if stop_file:
      ioloop.PeriodicCallback(self.check_stop_file, 5000).start()
コード例 #8
0
ファイル: pipeline.py プロジェクト: tylerknott/seesaw-kit
    def __init__(self, *tasks):
        self.cwd = os.getcwd()
        self.data_dir = os.path.join(self.cwd, "data")
        self.on_start_item = Event()
        self.on_complete_item = Event()
        self.on_fail_item = Event()
        self.on_cancel_item = Event()
        self.on_finish_item = Event()
        self.on_cleanup = Event()
        self.on_stop_requested = Event()
        self.on_stop_canceled = Event()
        self.project = None

        self.items_in_pipeline = set()
        self.tasks = []
        for task in tasks:
            self.add_task(task)
コード例 #9
0
ファイル: warrior.py プロジェクト: km0420j/seesaw-kit
    def __init__(self,
                 projects_dir,
                 data_dir,
                 warrior_hq_url,
                 real_shutdown=False,
                 keep_data=False):
        if not os.access(projects_dir, os.W_OK):
            raise Exception("Couldn't write to projects directory: %s" %
                            projects_dir)
        if not os.access(data_dir, os.W_OK):
            raise Exception("Couldn't write to data directory: %s" % data_dir)

        self.projects_dir = projects_dir
        self.data_dir = data_dir
        self.warrior_hq_url = warrior_hq_url
        self.real_shutdown = real_shutdown
        self.keep_data = keep_data

        # disable the password prompts
        self.gitenv = dict(
            list(os.environ.items()) + list({
                'GIT_ASKPASS': '******',
                'SSH_ASKPASS': '******'
            }.items()))

        self.warrior_id = StringConfigValue(
            name="warrior_id",
            title="Warrior ID",
            description="The unique number of your warrior instance.",
            editable=False)
        self.selected_project_config_value = StringConfigValue(
            name="selected_project",
            title="Selected project",
            description="The project (to be continued when the warrior "
            "restarts).",
            default="none",
            editable=False)
        self.downloader = StringConfigValue(
            name="downloader",
            title="Your nickname",
            description="We use your nickname to show your results on our "
            "tracker. Letters and numbers only.",
            regex="^[-_a-zA-Z0-9]{3,30}$",
            advanced=False)
        self.concurrent_items = NumberConfigValue(
            name="concurrent_items",
            title="Concurrent items",
            description="How many items should the warrior download at a "
            "time? (Max: 6)",
            min=1,
            max=6,
            default=2)
        self.http_username = StringConfigValue(
            name="http_username",
            title="HTTP username",
            description="Enter a username to protect the web interface, "
            "or leave empty.",
            default="")
        self.http_password = StringConfigValue(
            name="http_password",
            title="HTTP password",
            description="Enter a password to protect the web interface, "
            "or leave empty.",
            default="")

        self.config_manager = ConfigManager(
            os.path.join(projects_dir, "config.json"))
        self.config_manager.add(self.warrior_id)
        self.config_manager.add(self.selected_project_config_value)
        self.config_manager.add(self.downloader)
        self.config_manager.add(self.concurrent_items)
        self.config_manager.add(self.http_username)
        self.config_manager.add(self.http_password)

        self.bandwidth_monitor = BandwidthMonitor("eth0")
        self.bandwidth_monitor.update()

        self.runner = Runner(concurrent_items=self.concurrent_items,
                             keep_data=self.keep_data)
        self.runner.on_finish += self.handle_runner_finish

        self.current_project_name = None
        self.current_project = None

        self.selected_project = None

        self.projects = {}
        self.installed_projects = set()
        self.failed_projects = set()

        self.on_projects_loaded = Event()
        self.on_project_installing = Event()
        self.on_project_installed = Event()
        self.on_project_installation_failed = Event()
        self.on_project_refresh = Event()
        self.on_project_selected = Event()
        self.on_status = Event()
        self.on_broadcast_message_received = Event()

        self.http_client = AsyncHTTPClient()

        self.installing = False
        self.shut_down_flag = False
        self.reboot_flag = False

        io_loop = ioloop.IOLoop.instance()

        def update_warror_callback():
            io_loop.add_future(self.update_warrior_hq(),
                               lambda fut: fut.result())

        def update_project_callback():
            io_loop.add_future(self.update_project(), lambda fut: fut.result())

        self.hq_updater = ioloop.PeriodicCallback(update_warror_callback,
                                                  10 * 60 * 1000)
        self.project_updater = ioloop.PeriodicCallback(update_project_callback,
                                                       30 * 60 * 1000)
        self.forced_reboot_timeout = None

        self.lat_lng = None
        self.find_lat_lng()

        self.install_output = None
        self.broadcast_message = None
        self.contacting_hq_failed = False
コード例 #10
0
class Item(ItemData):
    '''A thing, or work unit, that needs to be downloaded.

    It has properties that are filled by the :class:`Task`.

    An Item behaves like a mutable mapping.

    .. note::
        State belonging to a item should be stored on the actual item
        itself. That is, do not store variables onto a :class:`Task` unless
        you know what you are doing.
    '''

    class ItemState(object):
        '''State of the item.'''
        running = "running"
        canceled = "canceled"
        completed = "completed"
        failed = "failed"

    class TaskStatus(object):
        '''Status of happened on a task.'''
        running = "running"
        completed = "completed"
        failed = "failed"

    def __init__(self, pipeline, item_id, item_number,
                 keep_data=False, prepare_data_directory=True,
                 **kwargs):
        super(Item, self).__init__(**kwargs)

        self._pipeline = pipeline
        self._item_id = item_id
        self._item_number = item_number
        self._keep_data = keep_data
        self.may_be_canceled = False

        self._item_state = self.ItemState.running
        self._task_status = {}
        self._start_time = time.time()
        self._end_time = None
        self._errors = []
        self._last_output = ""

        self.on_output = Event()
        self.on_error = Event()
        self.on_item_state = Event()
        self.on_task_status = Event()

        # Legacy events
        self.on_cancel = Event()
        self.on_complete = Event()
        self.on_fail = Event()
        self.on_finish = Event()
        self.on_item_state.handle(self._dispatch_legacy_events)

        if prepare_data_directory:
            self.prepare_data_directory()

    def __hash__(self):
        return hash(self._item_id)

    @property
    def item_state(self):
        return self._item_state

    @property
    def task_status(self):
        return self._task_status

    @property
    def start_time(self):
        return self._start_time

    @property
    def end_time(self):
        return self._end_time

    @property
    def canceled(self):
        return self._item_state == self.ItemState.canceled

    @property
    def completed(self):
        return self._item_state == self.ItemState.completed

    @property
    def failed(self):
        return self._item_state == self.ItemState.failed

    @property
    def finished(self):
        return self.canceled or self.completed or self.failed

    @property
    def pipeline(self):
        return self._pipeline

    @property
    def item_id(self):
        return self._item_id

    @property
    def item_number(self):
        return self._item_number

    def prepare_data_directory(self):
        dirname = os.path.join(self._pipeline.data_dir, self._item_id)
        self["data_dir"] = dirname
        if os.path.isdir(dirname):
            shutil.rmtree(dirname)
        os.makedirs(dirname)

    def clear_data_directory(self):
        if not self._keep_data:
            dirname = self["data_dir"]
            if os.path.isdir(dirname):
                shutil.rmtree(dirname)

    def log_output(self, data, full_line=True):
        if isinstance(data, seesaw.six.binary_type):
            try:
                data = data.decode('utf8', 'replace')
            except UnicodeError:
                data = data.decode('ascii', 'replace')

        if full_line and len(data) > 0:
            if data[0] != "\n" and len(self._last_output) > 0 and \
                    self._last_output[-1] != "\n":
                data = "\n" + data
            if data[-1] != "\n":
                data += "\n"
        self._last_output = data
        self.on_output(self, data)

    def log_error(self, task, *args):
        self._errors.append((task, args))
        self.on_error(self, task, *args)

    def set_task_status(self, task, status):
        if task in self._task_status:
            old_status = self._task_status[task]
        else:
            old_status = None
        if status != old_status:
            self._task_status[task] = status
            self.on_task_status(self, task, status, old_status)

    def cancel(self):
        assert not self.canceled
        self.clear_data_directory()
        self._item_state = self.ItemState.canceled
        self._end_time = time.time()
        self.on_item_state(self, self._item_state)

    def complete(self):
        assert not self.completed
        self.clear_data_directory()
        self._item_state = self.ItemState.completed
        self._end_time = time.time()
        self.on_item_state(self, self._item_state)

    def fail(self):
        assert not self.failed
        self.clear_data_directory()
        self._item_state = self.ItemState.failed
        self._end_time = time.time()
        self.on_item_state(self, self._item_state)

    def _dispatch_legacy_events(self, item, state):
        if state == self.ItemState.failed:
            self.on_fail(item)
        elif state == self.ItemState.completed:
            self.on_complete(item)
        elif state == self.ItemState.canceled:
            self.on_cancel(item)
        else:
            raise Exception('Unknown event')

        self.on_finish(item)

    def description(self):
        return "Item %s" % self.properties.get("item_name", "")

    def __str__(self):
        return "<Item '{0}' {1} {2}>".format(
            self.properties.get("item_name", ""),
            self._item_id, self._item_state
        )
コード例 #11
0
 def __init__(self, properties=None):
     super(ItemData, self).__init__()
     self._properties = properties or {}
     self.on_property = Event()