def server_side(config):
"""
Server implementation
"""
global file_count
# count of outstanding work items
eof_seen = False
terminate_count = mpi_size - 1
log_print("I'll be your server today on %s with a party of %i" % (os.getcwd(), mpi_size))
work_list = config.get(MPIS_CONFIG_SECT, "video_work_list")
fd = open(work_list, "r") # open work list
# The VCDStore in which we put deserialized elements.
work_dir = config.get(MPIS_CONFIG_SECT, "run_work_directory")
target_vcds = config.get(MPIS_CONFIG_SECT, "target_vcd_store")
vcd_store_overwrite = config.getboolean(MPIS_CONFIG_SECT, "vcd_store_overwrite")
vcd_store = VCDStore(db_root=work_dir, fs_db_path=target_vcds)
descr_mode = config.get(MPIS_CONFIG_SECT, "descriptor_mode")
while not (eof_seen and terminate_count <= 0):
log_print("Waiting for work request")
req_status = MPI.Status()
request = comm.recv(source=MPI.ANY_SOURCE, status=req_status)
log_print("Server request: [%s] from %i" % (request, req_status.source))
request = request.split()
command = request[0]
args = request[1:]
if command == "feed-me": # --- command ---
worker_fed = False
while not worker_fed:
#: :type: str
work_item = fd.readline()
if work_item:
work_item = work_item.strip() # strip trailing newline
# If the predicted result file already exists, then
# processing must have already occurred. Don't bother
# calling the module process function if this is the case.
element_file = get_vcd_element_file(descr_mode, work_item, work_dir)
if osp.isfile(element_file) and not vcd_store_overwrite:
log_print(
"Existing element file discovered for video "
"'%s', skipping assumed redundant processing "
"for video." % work_item
)
# no messages sent, cycle to next work item / EoF
else:
log_print(
"Sending work element %d, file %s to node %i" % (file_count, work_item, req_status.source)
)
comm.send("proc-file " + work_item, dest=req_status.source)
file_count += 1
worker_fed = True
else:
eof_seen = True
comm.send("die-sucka", dest=req_status.source)
log_print("*** sending terminate to %i" % req_status.source)
terminate_count -= 1
log_print("Terminate count: %i" % terminate_count)
worker_fed = True
elif command == "complete": # --- command ---
log_print("Worker %d completed a work item!" % req_status.source)
# Existence check of processed file handled before sending work to
# client, so just de-serialize and store whatever the client sends
# back after being told to process something (if anything is even
# sent back besides completion message)
if args: # may be empty I guess? if client sends back nothing
serialized_file = args[0] # expecting only one other argument
with open(serialized_file) as f:
elements = cPickle.load(f)
log_print("Attempting storage of elements")
for e in elements:
log_debug("-> %s", str(e))
try:
vcd_store.store_feature(elements, vcd_store_overwrite)
log_print("Storage successful")
except VCDErrors.VCDDuplicateFeatureError, ex:
log_warn(
"Attempted duplicate element insertion, "
"rolling back element insertion for dump -> %s\n"
"|--> (error: %s)" % (serialized_file, str(ex))
)
else:
log_print("No serialized objects returned from client, nothing " "to do.")
else:
log_print("Unrecognised command from node %d" % req_status.source)
def server_side(config):
"""
Server implementation
"""
global file_count
# count of outstanding work items
eof_seen = False
terminate_count = mpi_size - 1
log_print("I'll be your server today on %s with a party of %i" %
(os.getcwd(), mpi_size))
work_list = config.get(MPIS_CONFIG_SECT, 'video_work_list')
fd = open(work_list, 'r') # open work list
# The VCDStore in which we put deserialized elements.
work_dir = config.get(MPIS_CONFIG_SECT, 'run_work_directory')
target_vcds = config.get(MPIS_CONFIG_SECT, 'target_vcd_store')
vcd_store_overwrite = config.getboolean(MPIS_CONFIG_SECT,
'vcd_store_overwrite')
vcd_store = VCDStore(db_root=work_dir, fs_db_path=target_vcds)
descr_mode = config.get(MPIS_CONFIG_SECT, 'descriptor_mode')
while not (eof_seen and terminate_count <= 0):
log_print("Waiting for work request")
req_status = MPI.Status()
request = comm.recv(source=MPI.ANY_SOURCE, status=req_status)
log_print("Server request: [%s] from %i" %
(request, req_status.source))
request = request.split()
command = request[0]
args = request[1:]
if command == "feed-me": # --- command ---
worker_fed = False
while not worker_fed:
#: :type: str
work_item = fd.readline()
if work_item:
work_item = work_item.strip() # strip trailing newline
# If the predicted result file already exists, then
# processing must have already occurred. Don't bother
# calling the module process function if this is the case.
element_file = get_vcd_element_file(
descr_mode, work_item, work_dir)
if osp.isfile(element_file) and not vcd_store_overwrite:
log_print(
"Existing element file discovered for video "
"'%s', skipping assumed redundant processing "
"for video." % work_item)
# no messages sent, cycle to next work item / EoF
else:
log_print(
"Sending work element %d, file %s to node %i" %
(file_count, work_item, req_status.source))
comm.send("proc-file " + work_item,
dest=req_status.source)
file_count += 1
worker_fed = True
else:
eof_seen = True
comm.send('die-sucka', dest=req_status.source)
log_print("*** sending terminate to %i" %
req_status.source)
terminate_count -= 1
log_print("Terminate count: %i" % terminate_count)
worker_fed = True
elif command == "complete": # --- command ---
log_print("Worker %d completed a work item!" % req_status.source)
# Existence check of processed file handled before sending work to
# client, so just de-serialize and store whatever the client sends
# back after being told to process something (if anything is even
# sent back besides completion message)
if args: # may be empty I guess? if client sends back nothing
serialized_file = args[0] # expecting only one other argument
with open(serialized_file) as f:
elements = cPickle.load(f)
log_print("Attempting storage of elements")
for e in elements:
log_debug('-> %s', str(e))
try:
vcd_store.store_feature(elements, vcd_store_overwrite)
log_print("Storage successful")
except VCDErrors.VCDDuplicateFeatureError, ex:
log_warn("Attempted duplicate element insertion, "
"rolling back element insertion for dump -> %s\n"
"|--> (error: %s)" % (serialized_file, str(ex)))
else:
log_print(
"No serialized objects returned from client, nothing "
"to do.")
else:
log_print("Unrecognised command from node %d" % req_status.source)
### Validate / Initialize vars
if not opts.descriptor_name:
print "ERROR: No descriptor name given. We must have one!"
exit(1)
descr_name = opts.descriptor_name
if not opts.clip_id_file or not osp.isfile(opts.clip_id_file):
print "ERROR: Clip ID file not valid (given: '%s')" % opts.clip_id_file
exit(1)
if not opts.npy_file or not osp.isfile(opts.npy_file):
print "ERROR: Numpy data file not valid (given: '%s')" % opts.npy_file
exit(1)
if opts.store_file:
vcd_store = VCDStore(fs_db_path=opts.store_file)
else:
vcd_store = VCDStore()
overwrite = opts.overwrite
### Loading up clip IDs and numpy file, pairing parallel rows
print "Extracting clip IDs and features..."
clip_ids = map(int, (l.strip(' \t\n') for l in open(opts.clip_id_file).readlines()))
npy_matrix = np.load(opts.npy_file)
assert len(clip_ids) == npy_matrix.shape[0], \
"ERROR: Number of clip ids found did not match the numpy matrix size! " \
"(clip ids: %d) (np arrays: %d)" \
% (len(clip_ids), npy_matrix.shape[0])
print "-->", len(clip_ids)