def write_grouped_fofn_chunks(fofn_files, max_total_chunks, chunk_dir_name, chunk_json_path):
fofn_groups = _to_grouped_items_by_max_total_chunks(
fofn_files, max_total_chunks)
chunks = to_chunked_grouped_fofn(
fofn_groups, 'fofn_group', Constants.CHUNK_KEY_FOFN, Constants.CHUNK_KEY_FOFN_REPORT, chunk_dir_name)
IO.write_pipeline_chunks(
chunks, chunk_json_path, "Group Fofn created at {d}".format(d=datetime.datetime.now()))
return chunks
def write_grouped_fofn_chunks(fofn_files, max_total_chunks, chunk_dir_name,
chunk_json_path):
fofn_groups = _to_grouped_items_by_max_total_chunks(
fofn_files, max_total_chunks)
chunks = to_chunked_grouped_fofn(fofn_groups, 'fofn_group',
Constants.CHUNK_KEY_FOFN,
Constants.CHUNK_KEY_FOFN_REPORT,
chunk_dir_name)
IO.write_pipeline_chunks(
chunks, chunk_json_path,
"Group Fofn created at {d}".format(d=datetime.datetime.now()))
return chunks
def test_write_chunks(self):
def f(i):
return {"{c}movie_fofn_id".format(c=PipelineChunk.CHUNK_KEY_PREFIX): "/path/to_movie-{i}.fofn".format(i=i),
"{c}region_fofn_id".format(c=PipelineChunk.CHUNK_KEY_PREFIX): "/path/rgn_{i}.fofn".format(i=i)}
to_i = lambda i: "chunk-id-{i}".format(i=i)
to_p = lambda i: PipelineChunk(to_i(i), **f(i))
nchunks = 5
pipeline_chunks = [to_p(i) for i in xrange(nchunks)]
log.debug(pipeline_chunks)
tmp_name = get_temp_file("_chunk.json")
IO.write_pipeline_chunks(pipeline_chunks, tmp_name, "Example chunk file")
pchunks = IO.load_pipeline_chunks_from_json(tmp_name)
self.assertEquals(len(pchunks), nchunks)
def test_write_chunks(self):
def f(i):
return {
"{c}movie_fofn_id".format(c=PipelineChunk.CHUNK_KEY_PREFIX):
"/path/to_movie-{i}.fofn".format(i=i),
"{c}region_fofn_id".format(c=PipelineChunk.CHUNK_KEY_PREFIX):
"/path/rgn_{i}.fofn".format(i=i)
}
to_i = lambda i: "chunk-id-{i}".format(i=i)
to_p = lambda i: PipelineChunk(to_i(i), **f(i))
nchunks = 5
pipeline_chunks = [to_p(i) for i in xrange(nchunks)]
log.debug(pipeline_chunks)
tmp_name = get_temp_file("_chunk.json")
IO.write_pipeline_chunks(pipeline_chunks, tmp_name,
"Example chunk file")
pchunks = IO.load_pipeline_chunks_from_json(tmp_name)
self.assertEquals(len(pchunks), nchunks)
def write_chunks_to_json(chunks, chunk_file):
log.debug("Wrote {n} chunks to {f}.".format(n=len(chunks), f=chunk_file))
IO.write_pipeline_chunks(
chunks, chunk_file,
"Chunks written at {d}".format(d=datetime.datetime.now()))
return 0
def write_chunks_to_json(chunks, chunk_file):
log.debug("Wrote {n} chunks to {f}.".format(n=len(chunks), f=chunk_file))
IO.write_pipeline_chunks(
chunks, chunk_file, "Chunks written at {d}".format(d=datetime.datetime.now()))
return 0
