Exemple #1
0
def server_side(config):
    """
    Server implementation
    """
    global file_count

    # count of outstanding work items
    eof_seen = False
    terminate_count = mpi_size - 1

    log_print("I'll be your server today on %s with a party of %i" % (os.getcwd(), mpi_size))

    work_list = config.get(MPIS_CONFIG_SECT, "video_work_list")
    fd = open(work_list, "r")  # open work list

    # The VCDStore in which we put deserialized elements.
    work_dir = config.get(MPIS_CONFIG_SECT, "run_work_directory")
    target_vcds = config.get(MPIS_CONFIG_SECT, "target_vcd_store")
    vcd_store_overwrite = config.getboolean(MPIS_CONFIG_SECT, "vcd_store_overwrite")
    vcd_store = VCDStore(db_root=work_dir, fs_db_path=target_vcds)

    descr_mode = config.get(MPIS_CONFIG_SECT, "descriptor_mode")

    while not (eof_seen and terminate_count <= 0):
        log_print("Waiting for work request")
        req_status = MPI.Status()
        request = comm.recv(source=MPI.ANY_SOURCE, status=req_status)
        log_print("Server request: [%s] from %i" % (request, req_status.source))

        request = request.split()
        command = request[0]
        args = request[1:]

        if command == "feed-me":  # --- command ---
            worker_fed = False
            while not worker_fed:
                #: :type: str
                work_item = fd.readline()
                if work_item:
                    work_item = work_item.strip()  # strip trailing newline
                    # If the predicted result file already exists, then
                    # processing must have already occurred. Don't bother
                    # calling the module process function if this is the case.
                    element_file = get_vcd_element_file(descr_mode, work_item, work_dir)
                    if osp.isfile(element_file) and not vcd_store_overwrite:
                        log_print(
                            "Existing element file discovered for video "
                            "'%s', skipping assumed redundant processing "
                            "for video." % work_item
                        )
                        # no messages sent, cycle to next work item / EoF
                    else:
                        log_print(
                            "Sending work element %d, file %s to node %i" % (file_count, work_item, req_status.source)
                        )
                        comm.send("proc-file " + work_item, dest=req_status.source)
                        file_count += 1
                        worker_fed = True
                else:
                    eof_seen = True
                    comm.send("die-sucka", dest=req_status.source)
                    log_print("*** sending terminate to %i" % req_status.source)
                    terminate_count -= 1
                    log_print("Terminate count: %i" % terminate_count)
                    worker_fed = True

        elif command == "complete":  # --- command ---
            log_print("Worker %d completed a work item!" % req_status.source)

            # Existence check of processed file handled before sending work to
            # client, so just de-serialize and store whatever the client sends
            # back after being told to process something (if anything is even
            # sent back besides completion message)
            if args:  # may be empty I guess? if client sends back nothing
                serialized_file = args[0]  # expecting only one other argument
                with open(serialized_file) as f:
                    elements = cPickle.load(f)
                log_print("Attempting storage of elements")
                for e in elements:
                    log_debug("-> %s", str(e))
                try:
                    vcd_store.store_feature(elements, vcd_store_overwrite)
                    log_print("Storage successful")
                except VCDErrors.VCDDuplicateFeatureError, ex:
                    log_warn(
                        "Attempted duplicate element insertion, "
                        "rolling back element insertion for dump -> %s\n"
                        "|--> (error: %s)" % (serialized_file, str(ex))
                    )
            else:
                log_print("No serialized objects returned from client, nothing " "to do.")

        else:
            log_print("Unrecognised command from node %d" % req_status.source)
Exemple #2
0
def server_side(config):
    """
    Server implementation
    """
    global file_count

    # count of outstanding work items
    eof_seen = False
    terminate_count = mpi_size - 1

    log_print("I'll be your server today on %s with a party of %i" %
              (os.getcwd(), mpi_size))

    work_list = config.get(MPIS_CONFIG_SECT, 'video_work_list')
    fd = open(work_list, 'r')  # open work list

    # The VCDStore in which we put deserialized elements.
    work_dir = config.get(MPIS_CONFIG_SECT, 'run_work_directory')
    target_vcds = config.get(MPIS_CONFIG_SECT, 'target_vcd_store')
    vcd_store_overwrite = config.getboolean(MPIS_CONFIG_SECT,
                                            'vcd_store_overwrite')
    vcd_store = VCDStore(db_root=work_dir, fs_db_path=target_vcds)

    descr_mode = config.get(MPIS_CONFIG_SECT, 'descriptor_mode')

    while not (eof_seen and terminate_count <= 0):
        log_print("Waiting for work request")
        req_status = MPI.Status()
        request = comm.recv(source=MPI.ANY_SOURCE, status=req_status)
        log_print("Server request: [%s] from %i" %
                  (request, req_status.source))

        request = request.split()
        command = request[0]
        args = request[1:]

        if command == "feed-me":  # --- command ---
            worker_fed = False
            while not worker_fed:
                #: :type: str
                work_item = fd.readline()
                if work_item:
                    work_item = work_item.strip()  # strip trailing newline
                    # If the predicted result file already exists, then
                    # processing must have already occurred. Don't bother
                    # calling the module process function if this is the case.
                    element_file = get_vcd_element_file(
                        descr_mode, work_item, work_dir)
                    if osp.isfile(element_file) and not vcd_store_overwrite:
                        log_print(
                            "Existing element file discovered for video "
                            "'%s', skipping assumed redundant processing "
                            "for video." % work_item)
                        # no messages sent, cycle to next work item / EoF
                    else:
                        log_print(
                            "Sending work element %d, file %s to node %i" %
                            (file_count, work_item, req_status.source))
                        comm.send("proc-file " + work_item,
                                  dest=req_status.source)
                        file_count += 1
                        worker_fed = True
                else:
                    eof_seen = True
                    comm.send('die-sucka', dest=req_status.source)
                    log_print("*** sending terminate to %i" %
                              req_status.source)
                    terminate_count -= 1
                    log_print("Terminate count: %i" % terminate_count)
                    worker_fed = True

        elif command == "complete":  # --- command ---
            log_print("Worker %d completed a work item!" % req_status.source)

            # Existence check of processed file handled before sending work to
            # client, so just de-serialize and store whatever the client sends
            # back after being told to process something (if anything is even
            # sent back besides completion message)
            if args:  # may be empty I guess? if client sends back nothing
                serialized_file = args[0]  # expecting only one other argument
                with open(serialized_file) as f:
                    elements = cPickle.load(f)
                log_print("Attempting storage of elements")
                for e in elements:
                    log_debug('-> %s', str(e))
                try:
                    vcd_store.store_feature(elements, vcd_store_overwrite)
                    log_print("Storage successful")
                except VCDErrors.VCDDuplicateFeatureError, ex:
                    log_warn("Attempted duplicate element insertion, "
                             "rolling back element insertion for dump -> %s\n"
                             "|--> (error: %s)" % (serialized_file, str(ex)))
            else:
                log_print(
                    "No serialized objects returned from client, nothing "
                    "to do.")

        else:
            log_print("Unrecognised command from node %d" % req_status.source)
Exemple #3
0
### Validate / Initialize vars
if not opts.descriptor_name:
    print "ERROR: No descriptor name given. We must have one!"
    exit(1)
descr_name = opts.descriptor_name

if not opts.clip_id_file or not osp.isfile(opts.clip_id_file):
    print "ERROR: Clip ID file not valid (given: '%s')" % opts.clip_id_file
    exit(1)
if not opts.npy_file or not osp.isfile(opts.npy_file):
    print "ERROR: Numpy data file not valid (given: '%s')" % opts.npy_file
    exit(1)

if opts.store_file:
    vcd_store = VCDStore(fs_db_path=opts.store_file)
else:
    vcd_store = VCDStore()

overwrite = opts.overwrite


### Loading up clip IDs and numpy file, pairing parallel rows
print "Extracting clip IDs and features..."
clip_ids = map(int, (l.strip(' \t\n') for l in open(opts.clip_id_file).readlines()))
npy_matrix = np.load(opts.npy_file)
assert len(clip_ids) == npy_matrix.shape[0], \
    "ERROR: Number of clip ids found did not match the numpy matrix size! " \
    "(clip ids: %d) (np arrays: %d)" \
    % (len(clip_ids), npy_matrix.shape[0])
print "-->", len(clip_ids)