def grab_projection_files(self):
download_dir = os.path.join(config.scratch_detector_path(), self.PCA_TAR_BALL_NAME)
bucket = config.s3_detector_bucket()
logger.info('Downloading files from s3')
s3client.download_tarball(bucket, self.PCA_TAR_BALL_NAME, download_dir)
pca_file = os.path.join(download_dir, 'pca.xml')
assert os.path.exists(pca_file), pca_file
return pca_file
def __init__(self, video, video_path):
self.video = video
self.video_path = video_path
self.base_dir = tempfile.mkdtemp()
self.transcript_path = os.path.join(self.base_dir, 'transcript.txt')
self.model_dir = os.path.join(config.scratch_detector_path(), 'asr')
self.hmm = os.path.join(self.model_dir, 'hmm', 'hub4wsj_sc_8k')
self.lm = os.path.join(self.model_dir, 'lm', 'hub4.5000.DMP')
self.dic = os.path.join(self.model_dir, 'lm', 'cmu07a.dic')
def create_named_pipes(self):
dir_name = 'named_pipes_%s' % uuid.uuid1()
named_pipe_dir = os.path.join(config.scratch_detector_path(), dir_name)
try:
os.makedirs(named_pipe_dir)
except OSError as e:
assert os.path.exists(named_pipe_dir), e
# We write page text and read back topics
self.output_pipe = os.path.join(named_pipe_dir, 'text.fifo')
self.input_pipe = os.path.join(named_pipe_dir, 'topic.fifo')
os.mkfifo(self.input_pipe)
os.mkfifo(self.output_pipe)
def start_server(self):
if self.server is not None:
return
self.current_version = TextDetectionVersion.get_latest_version_timestamp(
detector_type='topic_model', return_int=True)
assert self.current_version is not None, "No version!"
model_file_dir_name = 'classifier_model_files_%d_%s' % (
self.current_version, uuid.uuid1())
self.model_files_dir = os.path.join(config.scratch_detector_path(),
model_file_dir_name)
self.grab_s3_files()
self.load_category_data(
os.path.join(self.model_files_dir, "category_info.pickle"))
self.create_named_pipes()
self.start_server_process()
def local_dir(self):
destination = self.tarball_basename
if hasattr(self, 'updated_at'):
date_str = self.updated_at.strftime('%Y-%m-%d-%H-%M-%S')
destination += '_' + date_str
return os.path.join(config.scratch_detector_path(), destination)