コード例 #1
0
def init():
    """
    Gloabl Initialization routine for task
    """

    """ Create SharedState and give it runf """
    state = SharedState(task_name)
    state.runf = runf
    state.taskfolder = task_folder
    state.statepath = task_statepath

    if newstart:
        print('new start...')
        initial_idset = set(util.get_initial_ids())
        existing_idset = set()
        # initial_idset = set(config.get_initial_ids())
        # existing_idset = config.get_existing_idset()
        new_idset = initial_idset.difference(existing_idset)

        state.add_idset(new_idset)
        state.add_idset(existing_idset)

        """ Necessary step, initialize the task queue """
        for job in new_idset:
            state.queue.put(job)

        print('actually # ids used : %d' % len(new_idset))

    else:
        print('resuming from saved state...')
        state.fromfile(task_statepath)

    return state
コード例 #2
0
def init():
    """
    Gloabl Initialization routine for task
    """

    """ general initialization """
    socket.setdefaulttimeout(config.socket_timeout)

    """ Create SharedState and give it runf """
    state = SharedState(task_name)
    state.runf = runf
    state.taskfolder = task_folder
    state.statepath = task_statepath

    with open('./data/ios-home-res.info') as f:
        li = f.readlines()
    urlli = [line.strip().split()[1] for line in li]

    initial_idset = set(sum([list(makeurls_with_letters(caturl)) for caturl in urlli], []))
    print('size of initial set %d ' % len(initial_idset))

    """ Necessary step, initialize the task queue """
    for job in initial_idset:
        state.queue.put(job)

    return state
コード例 #3
0
def init():
    """
    Gloabl Initialization routine for task
    """
    print("Init -- download user review for selected apps")

    """ general initialization """

    """ Create SharedState and give it runf """
    state = SharedState(task_name)
    state.runf = runf
    state.taskfolder = task_folder
    state.statepath = task_statepath

    if newstart:
        print('new start...')

        """todo initial id set should be from the 5000 selected developers """
        initial_idset = set(get_selected_appids_kai())
        print('# new ids is : %d' % len(initial_idset))
        state.add_idset(initial_idset)

        """ Necessary step, initialize the task queue """
        for job in initial_idset:
            state.queue.put(job)

    else:
        print('resuming from saved state...')
        state.fromfile(task_statepath)

    return state
コード例 #4
0
def init():
    # socket.setdefaulttimeout(config.timeout)

    state = SharedState('devhomes')
    # this whole file would be "interpreted" before running this line
    state.runf = runf

    """
    devinfo is defined as 3 tuple (devid, url, toplevel)
    """
    devinfoli = config.get_devid_devhome()
    devinfoli = [(i[0], i[1], i[1] != '') for i in devinfoli]
    for devinfo in devinfoli:
        state.queue.put(devinfo)

    return state
コード例 #5
0
ファイル: task_template.py プロジェクト: zhicai/scraper
def init():
    """
    Gloabl Initialization routine for task
    """

    """ general initialization """
    socket.setdefaulttimeout(config.timeout)

    """ Create SharedState and give it runf """
    state = SharedState(task_name)
    state.counter = 0
    state.runf = runf

    """ Necessary step, initialize the task queue """
    for _ in range(1000):
        state.queue.put(None)

    return state