def test_generated_sfTCP():
graph, existence_for_node_time, connectivity_demand = generate_scale_free_graph(
num_nodes=1000, max_time=3, active_time_percent=1)
model, edge_variables = generate_TCP_model(graph, existence_for_node_time,
connectivity_demand)
NUM_ITERATIONS = 10
vertex_usage_count = {}
solutions = []
start_time = python_time.time()
for _ in range(NUM_ITERATIONS):
subgraph = solve_TCP_instance(model, graph, edge_variables)
if not subgraph:
break
model = add_optimal_solution_constraint(model,
subgraph,
edge_variables,
additional_constraint=2)
solutions.append(subgraph)
for vertex in subgraph.nodes():
if vertex in vertex_usage_count:
vertex_usage_count[vertex] += 1
else:
vertex_usage_count[vertex] = 1
end_time = python_time.time()
days, hours, minutes, seconds = execution_time(start_time, end_time)
print(
'Solving for multiple solutions %s days, %s hours, %s minutes, %s seconds'
% (days, hours, minutes, seconds))
return graph, vertex_usage_count
def solve_TCP_instance(model,
graph,
edge_variables,
detailed_output=True,
time_output=False):
"""
Given a simple TCP problem instance, returns a minimum weight subgraph that satisfies the demand.
:param graph: a directed graph with attribute 'weight' on all edges
:param model: a Gurobi model to be optimized
:param edge_variables: a dictionary of variables corresponding to the variables d_v,w
:param detailed_output: flag which when True will print the edges in the optimal subgraph
:param time_output: flag which when True will return the time taken to obtain result rather than the optimal subgraph
:return: a optimal subgraph containing the path if the solution is Optimal, else None
"""
start_time = python_time.time()
# SOLVE AND RECOVER SOLUTION
print(
'-----------------------------------------------------------------------'
)
model.optimize()
print "Solution Count: " + str(model.SolCount)
subgraph = retreive_and_print_subgraph(model, graph, edge_variables,
detailed_output)
end_time = python_time.time()
days, hours, minutes, seconds = execution_time(start_time, end_time)
print('sDCP solving took %s days, %s hours, %s minutes, %s seconds' %
(days, hours, minutes, seconds))
# Return solution iff found
if time_output:
return end_time - start_time
return subgraph if model.status == GRB.status.OPTIMAL else None
def solve_TCP_instance(graph, existence_for_node_time, connectivity_demand, detailed_output=True, time_output=False):
"""
Given a simple TCP problem instance, returns a minimum weight subgraph that satisfies the demand.
:param graph: a directed graph with attribute 'weight' on all edges
:param existence_for_node_time: a dictionary from (node, time) to existence {True, False}
:param connectivity_demand: a connectivity demand (source, demand)
:param detailed_output: flag which when True will print the edges in the optimal subgraph
:param time_output: flag which when True will return the time taken to obtain result rather than the optimal subgraph
:return:
"""
start_time = python_time.time()
# MODEL SETUP
# Infer a list of times
times = list(set([node_time[1] for node_time in existence_for_node_time.keys()]))
# Sources get +1 sourceflow, destinations get -1, other nodes 0
sourceflow = {(v, t): 0 for v in graph.nodes_iter() for t in times}
source, destination = connectivity_demand
sourceflow[source, 0] = 1
sourceflow[destination, max(times)] = -1
# Create empty optimization model
model = Model('temporal_connectivity')
# Create variables d_{uvtt'}
edge_time_variables = {}
for t in times:
for t_prime in times:
for u, v in graph.edges_iter():
edge_time_variables[u, v, t, t_prime] = model.addVar(vtype=GRB.BINARY, name='edge_time_%s_%s_%s_%s' % (u, v, t, t_prime))
# Create variables d_{uv}
edge_variables = {}
for u, v in graph.edges_iter():
edge_variables[u, v] = model.addVar(vtype=GRB.BINARY, name='edge_%s_%s' % (u, v))
model.update()
# CONSTRAINTS
# Edge decision constraints (an edge is chosen if it is chosen at any time)
for t in times:
for t_prime in times:
for u, v in graph.edges_iter():
model.addConstr(edge_variables[u, v] >= edge_time_variables[u, v, t, t_prime])
# Existence constraints (can only route flow through active nodes)
for t in times:
for t_prime in times:
for u, v in graph.edges_iter():
model.addConstr(edge_time_variables[u, v, t, t_prime] <= existence_for_node_time[u, t])
model.addConstr(edge_time_variables[u, v, t, t_prime] <= existence_for_node_time[v, t_prime])
for t in times:
for t_prime in times:
if t != t_prime and t+1 != t_prime:
model.addConstr(edge_time_variables[u, v, t, t_prime] == 0)
# Flow conservation constraints
for t in times:
for v in graph.nodes_iter():
if t != 0 and t != max(times):
model.addConstr(
quicksum(edge_time_variables[u, v, t-1, t] for u in graph.predecessors_iter(v)) +
quicksum(edge_time_variables[u, v, t, t] for u in graph.predecessors_iter(v)) +
sourceflow[v, t] ==
quicksum(edge_time_variables[v, w, t, t] for w in graph.successors_iter(v)) +
quicksum(edge_time_variables[v, w, t, t+1] for w in graph.successors_iter(v))
)
if t == 0:
model.addConstr(
quicksum(edge_time_variables[u, v, t, t] for u in graph.predecessors_iter(v)) +
sourceflow[v, t] ==
quicksum(edge_time_variables[v, w, t, t] for w in graph.successors_iter(v)) +
quicksum(edge_time_variables[v, w, t, t + 1] for w in graph.successors_iter(v))
)
if t == max(times):
model.addConstr(
quicksum(edge_time_variables[u, v, t - 1, t] for u in graph.predecessors_iter(v)) +
quicksum(edge_time_variables[u, v, t, t] for u in graph.predecessors_iter(v)) +
sourceflow[v, t] ==
quicksum(edge_time_variables[v, w, t, t] for w in graph.successors_iter(v))
)
# OBJECTIVE
# Minimize total path weight
objective_expression = quicksum(edge_variables[u, v] * graph[u][v]['weight'] for u, v in graph.edges_iter())
model.setObjective(objective_expression, GRB.MINIMIZE)
# SOLVE AND RECOVER SOLUTION
print('-----------------------------------------------------------------------')
model.optimize()
subgraph = retreive_and_print_subgraph(model, graph, edge_variables, detailed_output)
end_time = python_time.time()
days, hours, minutes, seconds = execution_time(start_time, end_time)
print('sDCP solving took %s days, %s hours, %s minutes, %s seconds' % (days, hours, minutes, seconds))
# Return solution iff found
if time_output:
return end_time - start_time
return subgraph if model.status == GRB.status.OPTIMAL else None
def afire_dispatcher(afire_home, afire_data_dict, afire_options):
"""
Dispatch one or more Active Fires jobs to the multiprocessing pool, and report back the final
job statuses.
"""
# Construct a list of task dicts...
granule_id_list = sorted(afire_data_dict.keys())
afire_tasks = []
for granule_id in granule_id_list:
args = {
'granule_dict': afire_data_dict[granule_id],
'afire_home': afire_home,
'afire_options': afire_options
}
afire_tasks.append(args)
# Setup the processing pool
cpu_count = multiprocessing.cpu_count()
LOG.info('There are {} available CPUs'.format(cpu_count))
requested_cpu_count = afire_options['num_cpu']
if requested_cpu_count is not None:
LOG.info('We have requested {} {}'.format(
requested_cpu_count,
"CPU" if requested_cpu_count == 1 else "CPUs"))
if requested_cpu_count > cpu_count:
LOG.warn(
'{} requested CPUs is greater than available, using {}'.format(
requested_cpu_count, cpu_count))
cpus_to_use = cpu_count
else:
cpus_to_use = requested_cpu_count
else:
cpus_to_use = cpu_count
LOG.info('We are using {}/{} available CPUs'.format(
cpus_to_use, cpu_count))
pool = multiprocessing.Pool(cpus_to_use)
# Submit the Active Fire tasks to the processing pool
timeout = 9999999
result_list = []
start_time = time.time()
LOG.info("Submitting {} Active Fire {} to the pool...".format(
len(afire_tasks), "task" if len(afire_tasks) == 1 else "tasks"))
result_list = pool.map_async(afire_submitter, afire_tasks).get(timeout)
end_time = time.time()
total_afire_time = execution_time(start_time, end_time)
LOG.debug("afire execution took {:9.6f} seconds".format(
total_afire_time['delta']))
LOG.info(
"Active Fire execution took {} days, {} hours, {} minutes, {:8.6f} seconds"
.format(total_afire_time['days'], total_afire_time['hours'],
total_afire_time['minutes'], total_afire_time['seconds']))
LOG.info('')
rc_exe_dict = {}
rc_problem_dict = {}
# Loop through each of the Active Fire results collect error information
for result in result_list:
granule_id, afire_rc, problem_rc, exe_out = result
LOG.debug(">>> granule_id {}: afire_rc = {}, problem_rc = {}".format(
granule_id, afire_rc, problem_rc))
# Did the actual afire binary succeed?
rc_exe_dict[granule_id] = afire_rc
rc_problem_dict[granule_id] = problem_rc
return rc_exe_dict, rc_problem_dict
def afire_submitter(args):
'''
This routine encapsulates the single unit of work, multiple instances of which are submitted to
the multiprocessing queue. It takes as input whatever is required to complete the work unit,
and returns return values and output logging from the external process.
'''
# This try block wraps all code in this worker function, to capture any exceptions.
try:
granule_dict = args['granule_dict']
afire_home = args['afire_home']
afire_options = args['afire_options']
granule_id = granule_dict['granule_id']
run_dir = granule_dict['run_dir']
cmd = granule_dict['cmd']
work_dir = afire_options['work_dir']
env_vars = {}
rc_exe = 0
rc_problem = 0
exe_out = "Finished the Active Fires granule {}".format(granule_id)
LOG.debug("granule_id = {}".format(granule_id))
LOG.debug("run_dir = {}".format(run_dir))
LOG.debug("cmd = {}".format(cmd))
LOG.debug("work_dir = {}".format(work_dir))
LOG.debug("env_vars = {}".format(env_vars))
current_dir = os.getcwd()
LOG.info("Processing granule_id {}...".format(granule_id))
# Create the run dir for this input file
log_idx = 0
while True:
run_dir = pjoin(
work_dir, "{}_run_{}".format(granule_dict['run_dir'], log_idx))
if not exists(run_dir):
os.makedirs(run_dir)
break
else:
log_idx += 1
os.chdir(run_dir)
# Download and stage the required ancillary data for this input file
LOG.info("\tStaging the required ancillary data for granule_id {}...".
format(granule_id))
failed_ancillary = False
try:
rc_ancil, rc_ancil_dict, lwm_file = get_lwm(
afire_options, granule_dict)
failed_ancillary = True if rc_ancil != 0 else False
except Exception as err:
failed_ancillary = True
LOG.warn('\tProblem generating LWM for granule_id {}'.format(
granule_id))
LOG.error(err)
LOG.debug(traceback.format_exc())
# Run the active fire binary
if afire_options['ancillary_only']:
LOG.info(
'''\tAncillary only, skipping Active Fire execution for granule_id {}'''
.format(granule_id))
if failed_ancillary:
LOG.warn(
'\tAncillary granulation failed for granule_id {}'.format(
granule_id))
rc_problem = 1
os.chdir(current_dir)
elif failed_ancillary:
LOG.warn('\tAncillary granulation failed for granule_id {}'.format(
granule_id))
os.chdir(current_dir)
rc_problem = 1
else:
# Link the required files and directories into the work directory...
paths_to_link = [
pjoin(afire_home, 'vendor', afire_options['vfire_exe']),
lwm_file,
] + [
granule_dict[key]['file']
for key in afire_options['input_prefixes']
]
number_linked = link_files(run_dir, paths_to_link)
LOG.debug("\tWe are linking {} files to the run dir:".format(
number_linked))
for linked_files in paths_to_link:
LOG.debug("\t{}".format(linked_files))
# Contruct a dictionary of error conditions which should be logged.
error_keys = [
'FAILURE', 'failure', 'FAILED', 'failed', 'FAIL', 'fail',
'ERROR', 'error', 'ERR', 'err', 'ABORTING', 'aborting',
'ABORT', 'abort'
]
error_dict = {
x: {
'pattern': x,
'count_only': False,
'count': 0,
'max_count': None,
'log_str': ''
}
for x in error_keys
}
error_dict['error_keys'] = error_keys
start_time = time.time()
rc_exe, exe_out = execute_binary_captured_inject_io(
run_dir,
cmd,
error_dict,
log_execution=False,
log_stdout=False,
log_stderr=False,
**env_vars)
end_time = time.time()
afire_time = execution_time(start_time, end_time)
LOG.debug("\tafire execution of {} took {:9.6f} seconds".format(
granule_id, afire_time['delta']))
LOG.info(
"\tafire execution of {} took {} days, {} hours, {} minutes, {:8.6f} seconds"
.format(granule_id, afire_time['days'], afire_time['hours'],
afire_time['minutes'], afire_time['seconds']))
LOG.debug("\tGranule ID: {}, rc_exe = {}".format(
granule_id, rc_exe))
os.chdir(current_dir)
# Write the afire output to a log file, and parse it to determine the output
creation_dt = datetime.utcnow()
timestamp = creation_dt.isoformat()
logname = "{}_{}.log".format(run_dir, timestamp)
log_dir = dirname(run_dir)
logpath = pjoin(log_dir, logname)
logfile_obj = open(logpath, 'w')
for line in exe_out.splitlines():
logfile_obj.write(str(line) + "\n")
logfile_obj.close()
# Update the various file global attributes
try:
old_output_file = pjoin(run_dir, granule_dict['AFEDR']['file'])
creation_dt = granule_dict['creation_dt']
# Check whether the target AF text file exists, and remove it.
output_txt_file = '{}.txt'.format(splitext(old_output_file)[0])
if exists(output_txt_file):
LOG.debug('{} exists, removing.'.format(output_txt_file))
os.remove(output_txt_file)
#
# Update the attributes, moving to the end
#
if afire_options['i_band']:
# Update the I-band attributes, and write the fire data to a text file.
h5_file_obj = h5py.File(old_output_file, "a")
h5_file_obj.attrs.create(
'date_created', np.string_(creation_dt.isoformat()))
h5_file_obj.attrs.create('granule_id',
np.string_(granule_id))
history_string = 'CSPP Active Fires version: {}'.format(
afire_options['version'])
h5_file_obj.attrs.create('history',
np.string_(history_string))
h5_file_obj.attrs.create(
'Metadata_Link', np.string_(basename(old_output_file)))
h5_file_obj.attrs.create(
'id', np.string_(getURID(creation_dt)['URID']))
# Extract desired data from the NetCDF4 file, for output to the text file
nfire = h5_file_obj.attrs['FirePix'][0]
if int(nfire) > 0:
fire_datasets = [
'FP_latitude', 'FP_longitude', 'FP_T4',
'FP_confidence', 'FP_power'
]
fire_data = []
for dset in fire_datasets:
fire_data.append(h5_file_obj['/' + dset][:])
h5_file_obj.close()
# Check if there are any fire pixels, and write the associated fire data to
# a text file...
LOG.info("\tGranule {} has {} fire pixels".format(
granule_id, nfire))
if int(nfire) > 0:
Along_scan_pixel_dim = 0.375
Along_track_pixel_dim = 0.375
fire_pixel_res = [
Along_scan_pixel_dim, Along_track_pixel_dim
]
format_str = '''{0:13.8f}, {1:13.8f}, {2:13.8f}, {5:6.3f}, {6:6.3f},''' \
''' {3:4d}, {4:13.8f}'''
txt_file_header = \
'''# Active Fires I-band EDR\n''' \
'''#\n''' \
'''# source: {}\n''' \
'''# version: {}\n''' \
'''#\n''' \
'''# column 1: latitude of fire pixel (degrees)\n''' \
'''# column 2: longitude of fire pixel (degrees)\n''' \
'''# column 3: I04 brightness temperature of fire pixel (K)\n''' \
'''# column 4: Along-scan fire pixel resolution (km)\n''' \
'''# column 5: Along-track fire pixel resolution (km)\n''' \
'''# column 6: detection confidence ([7,8,9]->[lo,med,hi])\n''' \
'''# column 7: fire radiative power (MW)\n''' \
'''#\n# number of fire pixels: {}\n''' \
'''#'''.format(basename(old_output_file), history_string, nfire)
nasa_file = output_txt_file.replace('dev', 'dev_nasa')
if exists(nasa_file):
LOG.debug('{} exists, removing.'.format(nasa_file))
os.remove(nasa_file)
LOG.info("\tWriting output text file {}".format(
output_txt_file))
txt_file_obj = open(output_txt_file, 'x')
try:
txt_file_obj.write(txt_file_header + "\n")
for FP_latitude, FP_longitude, FP_R13, FP_confidence, FP_power in zip(
*fire_data):
fire_vars = [
FP_latitude, FP_longitude, FP_R13,
FP_confidence, FP_power
]
line = format_str.format(*(fire_vars +
fire_pixel_res))
txt_file_obj.write(line + "\n")
txt_file_obj.close()
except Exception:
txt_file_obj.close()
rc_problem = 1
LOG.warning(
"\tProblem writing Active fire text file: {}".
format(output_txt_file))
LOG.warn(traceback.format_exc())
else:
# Update the M-band attributes, and write the fire data to a text file.
nc_file_obj = Dataset(old_output_file,
"a",
format="NETCDF4")
setattr(nc_file_obj, 'date_created',
creation_dt.isoformat())
setattr(nc_file_obj, 'granule_id', granule_id)
history_string = 'CSPP Active Fires version: {}'.format(
afire_options['version'])
setattr(nc_file_obj, 'history', history_string)
setattr(nc_file_obj, 'Metadata_Link',
basename(old_output_file))
setattr(nc_file_obj, 'id', getURID(creation_dt)['URID'])
# Extract desired data from the NetCDF4 file, for output to the text file
nfire = len(nc_file_obj['Fire Pixels'].dimensions['nfire'])
if int(nfire) > 0:
fire_datasets = [
'FP_latitude', 'FP_longitude', 'FP_T13',
'FP_confidence', 'FP_power'
]
fire_data = []
for dset in fire_datasets:
fire_data.append(
nc_file_obj['Fire Pixels'].variables[dset][:])
nc_file_obj.close()
# Check if there are any fire pixels, and write the associated fire data to
# a text file...
LOG.info("\tGranule {} has {} fire pixels".format(
granule_id, nfire))
if int(nfire) > 0:
Along_scan_pixel_dim = 0.75
Along_track_pixel_dim = 0.75
fire_pixel_res = [
Along_scan_pixel_dim, Along_track_pixel_dim
]
format_str = '''{0:13.8f}, {1:13.8f}, {2:13.8f}, {5:6.3f}, {6:6.3f},''' \
''' {3:4d}, {4:13.8f}'''
txt_file_header = \
'''# Active Fires M-band EDR\n''' \
'''#\n''' \
'''# source: {}\n''' \
'''# version: {}\n''' \
'''#\n''' \
'''# column 1: latitude of fire pixel (degrees)\n''' \
'''# column 2: longitude of fire pixel (degrees)\n''' \
'''# column 3: M13 brightness temperature of fire pixel (K)\n''' \
'''# column 4: Along-scan fire pixel resolution (km)\n''' \
'''# column 5: Along-track fire pixel resolution (km)\n''' \
'''# column 6: detection confidence (%)\n''' \
'''# column 7: fire radiative power (MW)\n''' \
'''#\n# number of fire pixels: {}\n''' \
'''#'''.format(basename(old_output_file), history_string, nfire)
LOG.info("\tWriting output text file {}".format(
output_txt_file))
txt_file_obj = open(output_txt_file, 'x')
try:
txt_file_obj.write(txt_file_header + "\n")
for FP_latitude, FP_longitude, FP_T13, FP_confidence, FP_power in zip(
*fire_data):
fire_vars = [
FP_latitude, FP_longitude, FP_T13,
FP_confidence, FP_power
]
line = format_str.format(*(fire_vars +
fire_pixel_res))
txt_file_obj.write(line + "\n")
txt_file_obj.close()
except Exception:
txt_file_obj.close()
rc_problem = 1
LOG.warning(
"\tProblem writing Active fire text file: {}".
format(output_txt_file))
LOG.warn(traceback.format_exc())
except Exception:
rc_problem = 1
LOG.warning(
"\tProblem setting attributes in output file {}".format(
old_output_file))
LOG.debug(traceback.format_exc())
# Move output files to the work directory
LOG.debug("\tMoving output files from {} to {}".format(
run_dir, work_dir))
af_prefix = 'AFIMG' if afire_options['i_band'] else 'AFMOD'
af_suffix = 'nc' if afire_options[
'i_band'] else 'nc' # FIXME: NOAA should fix NC output for I-band!
outfiles = glob(pjoin(run_dir, '{}*.{}'.format(af_prefix, af_suffix))) \
+ glob(pjoin(run_dir, '{}*.txt'.format(af_prefix)))
for outfile in outfiles:
try:
shutil.move(outfile, work_dir)
except Exception:
rc_problem = 1
LOG.warning(
"\tProblem moving output {} from {} to {}".format(
outfile, run_dir, work_dir))
LOG.debug(traceback.format_exc())
# If no problems, remove the run dir
if (rc_exe == 0) and (rc_problem == 0) and afire_options['docleanup']:
cleanup([run_dir])
except Exception:
LOG.warn("\tGeneral warning for {}".format(granule_id))
LOG.debug(traceback.format_exc())
os.chdir(current_dir)
#raise
return [granule_id, rc_exe, rc_problem, exe_out]
def nagg_submitter(args):
'''
This routine encapsulates the single unit of work, multiple instances of which are submitted to
the multiprocessing queue. It takes as input whatever is required to complete the work unit,
and returns return values and output logging from the external process.
'''
# This try block wraps all code in this worker function, to capture any exceptions.
try:
afire_home = args['afire_home']
agg_input_file = args['agg_input_file']
unagg_inputs_dir = args['unagg_inputs_dir']
afire_options = args['afire_options']
work_dir = afire_options['work_dir']
env_vars = {}
rc_exe = 0
rc_problem = 0
exe_out = "Finished running nagg on the aggregated VIIRS file: {}".format(
agg_input_file)
LOG.debug("afire_home = {}".format(afire_home))
LOG.debug("agg_input_file = {}".format(agg_input_file))
LOG.debug("unagg_inputs_dir = {}".format(unagg_inputs_dir))
LOG.debug("work_dir = {}".format(work_dir))
LOG.debug("env_vars = {}".format(env_vars))
current_dir = os.getcwd()
LOG.info("Processing aggregated file {}...".format(agg_input_file))
os.chdir(unagg_inputs_dir)
#nagg_exe = os.path.join(afire_home, 'vendor/nagg')
nagg_exe = './nagg'
prefix = os.path.basename(agg_input_file).split('_')[0]
cmd_dict = {}
cmd_dict['GEO'] = '{} -S -g {} -n 1 -O cspp -D dev -d {} {}'.format(
nagg_exe, prefix, unagg_inputs_dir, agg_input_file)
cmd_dict[
'SVM'] = '{} -t {} -S -g no -n 1 -O cspp -D dev -d {} {}'.format(
nagg_exe, prefix, unagg_inputs_dir, agg_input_file)
if 'GMTCO' in os.path.basename(agg_input_file):
cmd = cmd_dict['GEO']
elif 'GITCO' in os.path.basename(agg_input_file):
cmd = cmd_dict['GEO']
elif 'SVM' in os.path.basename(agg_input_file):
cmd = cmd_dict['SVM']
elif 'SVI' in os.path.basename(agg_input_file):
cmd = cmd_dict['SVM']
elif 'IVCDB' in os.path.basename(agg_input_file):
cmd = cmd_dict['SVM']
else:
cmd = None
#cmd = '''sleep 0.5; echo "Running nagg on {0:}"; exit 0'''.format(
#os.path.basename(agg_input_file))
# Contruct a dictionary of error conditions which should be logged.
error_keys = [
'FAILURE', 'failure', 'FAILED', 'failed', 'FAIL', 'fail', 'ERROR',
'error', 'ERR', 'err', 'ABORTING', 'aborting', 'ABORT', 'abort'
]
error_dict = {
x: {
'pattern': x,
'count_only': False,
'count': 0,
'max_count': None,
'log_str': ''
}
for x in error_keys
}
error_dict['error_keys'] = error_keys
if cmd is not None:
start_time = time.time()
rc_exe, exe_out = execute_binary_captured_inject_io(
unagg_inputs_dir,
cmd,
error_dict,
log_execution=False,
log_stdout=False,
log_stderr=False,
**env_vars)
end_time = time.time()
nagg_time = execution_time(start_time, end_time)
LOG.debug("\tnagg execution of {} took {:9.6f} seconds".format(
os.path.basename(agg_input_file), nagg_time['delta']))
LOG.info(
"\tnagg execution for {} took {} days, {} hours, {} minutes, {:8.6f} seconds"
.format(os.path.basename(agg_input_file), nagg_time['days'],
nagg_time['hours'], nagg_time['minutes'],
nagg_time['seconds']))
LOG.debug("\tnagg({}), rc_exe = {}".format(
os.path.basename(agg_input_file), rc_exe))
else:
exe_out = '''Aggregated file {} cannot be unaggregated by nagg,''' \
''' unrecognized prefix {}.'''.format(prefix, os.path.basename(agg_input_file))
LOG.warn('\t' + exe_out)
# Write the afire output to a log file, and parse it to determine the output
creation_dt = datetime.utcnow()
timestamp = creation_dt.isoformat()
logname = "nagg_unaggregate-{}-{}.log".format(
os.path.basename(agg_input_file), timestamp)
logpath = os.path.join(unagg_inputs_dir, logname)
logfile_obj = open(logpath, 'w')
for line in exe_out.splitlines():
logfile_obj.write(line + "\n")
logfile_obj.close()
os.chdir(current_dir)
except Exception:
LOG.warn("\tGeneral warning for {}".format(
os.path.basename(agg_input_file)))
LOG.debug(traceback.format_exc())
os.chdir(current_dir)
raise
return [os.path.basename(agg_input_file), rc_exe, rc_problem, exe_out]
def unaggregate_inputs(afire_home, agg_input_files, afire_options):
'''
Create a dir for the unaggregated files in the work dir, and use nagg to unaggregate the
aggregated input files.
'''
unagg_inputs_dir = os.path.join(afire_options['work_dir'],
'unaggregated_inputs')
unagg_inputs_dir = create_dir(unagg_inputs_dir)
# Construct a list of task dicts...
nagg_tasks = []
for agg_input_file in agg_input_files:
args = {
'afire_home': afire_home,
'agg_input_file': agg_input_file,
'unagg_inputs_dir': unagg_inputs_dir,
'afire_options': afire_options
}
nagg_tasks.append(args)
# Link the nagg executable into the unaggregated inputs dir...
paths_to_link = [os.path.join(afire_home, 'vendor/nagg')]
number_linked = link_files(unagg_inputs_dir, paths_to_link)
LOG.debug(
"\tWe are linking {} files to the run dir:".format(number_linked))
for linked_files in paths_to_link:
LOG.debug("\t{}".format(linked_files))
# Setup the processing pool
cpu_count = multiprocessing.cpu_count()
LOG.debug('There are {} available CPUs'.format(cpu_count))
requested_cpu_count = afire_options['num_cpu']
if requested_cpu_count is not None:
LOG.debug('We have requested {} {}'.format(
requested_cpu_count,
"CPU" if requested_cpu_count == 1 else "CPUs"))
if requested_cpu_count > cpu_count:
LOG.warn(
'{} requested CPUs is greater than available, using {}'.format(
requested_cpu_count, cpu_count))
cpus_to_use = cpu_count
else:
cpus_to_use = requested_cpu_count
else:
cpus_to_use = cpu_count
LOG.debug('We are using {}/{} available CPUs'.format(
cpus_to_use, cpu_count))
pool = multiprocessing.Pool(cpus_to_use)
# Submit the Active Fire tasks to the processing pool
timeout = 9999999
result_list = []
start_time = time.time()
LOG.info("Submitting {} nagg {} to the pool...".format(
len(nagg_tasks), "task" if len(nagg_tasks) == 1 else "tasks"))
result_list = pool.map_async(nagg_submitter, nagg_tasks).get(timeout)
end_time = time.time()
total_afire_time = execution_time(start_time, end_time)
LOG.debug("Unaggregation took {:9.6f} seconds".format(
total_afire_time['delta']))
LOG.info(
"Unaggregation took {} days, {} hours, {} minutes, {:8.6f} seconds".
format(total_afire_time['days'], total_afire_time['hours'],
total_afire_time['minutes'], total_afire_time['seconds']))
# Loop through each of the Active Fire results collect error information
for result in result_list:
agg_input_file, nagg_rc, problem_rc, exe_out = result
LOG.debug(
">>> agg_input_file {}: nagg_rc = {}, problem_rc = {}".format(
agg_input_file, nagg_rc, problem_rc))
return unagg_inputs_dir
import random
from utils import execution_time
from bubblesort import bubble_sorter
from mergesort import merge_sorter
from insertsort import insertion_sort
from shellsort import shell_sort
from selectionsort import selection_sort
from quicksort import quick_sort
lst = random.sample(range(0, 100000), 40000)
function_lst = [
merge_sorter, insertion_sort, shell_sort, bubble_sorter, selection_sort
]
for f in function_lst:
f = execution_time(f)
f(lst)