def get_vcd_element_file(descr_mode, video_file, work_dir): """ Get standard path to VCDStoreElement serialized file for a given work directory, descriptor mode and video. If the containing directory to this file path did not exist, It will be created by return. -> ".../store_elements/<vpfx>/<vkey>/<dscr>.<key>.elements" where "<...>" is filled in with relevant material for given video and descriptor. """ prefix, key = VCDWorkerInterface.get_video_prefix(video_file) d = osp.join(get_vcd_element_dir(work_dir), descr_mode, prefix, key) d = VCDWorkerInterface.create_dir(d) f = '%s.%s.elements' % (descr_mode, key) return osp.join(d, f)
def get_vcd_element_file(descr_mode, video_file, work_dir): """ Get standard path to VCDStoreElement serialized file for a given work directory, descriptor mode and video. If the containing directory to this file path did not exist, It will be created by return. -> ".../store_elements/<vpfx>/<vkey>/<dscr>.<key>.elements" where "<...>" is filled in with relevant material for given video and descriptor. """ prefix, key = VCDWorkerInterface.get_video_prefix(video_file) d = osp.join(get_vcd_element_dir(work_dir), descr_mode, prefix, key) d = VCDWorkerInterface.create_dir(d) f = "%s.%s.elements" % (descr_mode, key) return osp.join(d, f)
# will wait until they exist before proceeding as they may want to use them # before the rank 0 node gets to creating them. # dirs_we_care_about = [ work_dir, log_dir, img_dir, get_vcd_element_dir(work_dir) ] if rank: # client node: wait till directories exist while not all([osp.isdir(d) for d in dirs_we_care_about]): time.sleep(0.1) else: # server node: create directories if they don't exist yet try: for d in dirs_we_care_about: VCDWorkerInterface.create_dir(d) except Exception, ex: print "ERROR - Exception occurred : %s" % str(ex) print "ERROR - Exception traceback:\n%s" % traceback.format_exc() comm.Abort() raise ### # Basic logging setup # log_fmt_str = '%(levelname)7s - %(asctime)s - %(name)s.%(funcName)s - ' \ '%(message)s' log_level = logging.WARNING - (10 * opts.verbosity) root_log = logging.getLogger() log_file_name = osp.join( log_dir,
# Making sure required directories exist. # # Due to parallelism, only the rank 0 node will create them. All other ranks # will wait until they exist before proceeding as they may want to use them # before the rank 0 node gets to creating them. # dirs_we_care_about = [work_dir, log_dir, img_dir, get_vcd_element_dir(work_dir)] if rank: # client node: wait till directories exist while not all([osp.isdir(d) for d in dirs_we_care_about]): time.sleep(0.1) else: # server node: create directories if they don't exist yet try: for d in dirs_we_care_about: VCDWorkerInterface.create_dir(d) except Exception, ex: print "ERROR - Exception occurred : %s" % str(ex) print "ERROR - Exception traceback:\n%s" % traceback.format_exc() comm.Abort() raise ### # Basic logging setup # log_fmt_str = "%(levelname)7s - %(asctime)s - %(name)s.%(funcName)s - " "%(message)s" log_level = logging.WARNING - (10 * opts.verbosity) root_log = logging.getLogger() log_file_name = osp.join(log_dir, "%s-vcd_scheduler-%04i.log" % (time.strftime("%Y_%j_%H%M%S"), rank)) log_formatter = logging.Formatter(fmt=log_fmt_str)