def worker(id): print('Worker', id, 'started.') while not taskQueue.empty(): clip_num = taskQueue.get() if clip_num is None: break clip = process_clip() np.savez_compressed(c.TRAIN_DIR_CLIPS + str(clip_num), clip) if (clip_num + 1) % 100 == 0: print('Worker %d: Processed %d clips' % (id, clip_num + 1)) print('Worker', id, 'finished.')
def process_training_data(num_clips): """ Processes random training clips from the full training data. Saves to TRAIN_DIR_CLIPS by default. :param num_clips: The number of clips to process. Default = 5000000 :warning: This can take a couple of hours to complete with large numbers of clips. """ num_prev_clips = len(glob(c.TRAIN_DIR_CLIPS + '*')) for clip_num in range(num_prev_clips, num_clips + num_prev_clips): clip = process_clip() np.savez_compressed(c.TRAIN_DIR_CLIPS + str(clip_num), clip) if (clip_num + 1) % 100 == 0: print ('Processed %d clips' % (clip_num + 1))
def process_training_data(num_clips): num_prev_clips = len(glob(c.TRAIN_DIR_CLIPS + '*')) for clip_num in xrange(num_prev_clips, num_clips + num_prev_clips): clip = process_clip() np.savez_compressed(c.TRAIN_DIR_CLIPS + str(clip_num), clip) if (clip_num + 1) % 100 == 0: print 'Processed %d clips' % (clip_num + 1)