def set_up(identification, base):
identification = common.fix_up_identification(identification)
outpath = common.trace_output_path(identification)
fn = 'kbdbg' + identification + '.n3'
base = 'file://' + base
this = "http://kbd.bg/run" + str(datetime.datetime.now()).replace(
':', '-').replace('.', '-').replace(' ', '-')
kbdbg_fn = common.kbdbg_file_name(identification, fn)
rules_fn = kbdbg_fn + '_rules'
subprocess.call(['rm', '-f', rules_fn])
os.system('mkdir -p ' + outpath)
return kbdbg_fn, rules_fn, identification, base, this, outpath
def run(start, end, workers):
global global_start, graph_name_start
global_start = start
redis_fn = redislite.Redis().db
if workers:
worker_pool = ProcessPoolExecutor(max_workers=workers)
runs_graph = Graph(sparqlstore.SPARQLStore(sparql_uri), default_graph)
graph_name_start = runs_graph.value(kbdbg.latest, kbdbg['is'],
any=False).toPython()
identification = fix_up_identification(graph_name_start)
step_to_submit = -1
for step_graph_uri in profile(
list, (Collection(runs_graph, URIRef(graph_name_start)), )):
step_to_submit += 1
if step_to_submit < start - 1:
log("skipping [" + str(step_to_submit) + ']')
continue
if step_to_submit > end and end != -1:
log("ending")
break
args = (identification, step_graph_uri, step_to_submit, redis_fn)
if not workers:
work(*args)
else:
log('submit ' + '[' + str(step_to_submit) + ']' +
' (queue size: ' + str(len(futures)) + ')')
if len(futures) > workers:
time.sleep(len(futures) - workers)
fut = worker_pool.submit(work, *args)
fut.step = step_to_submit
futures.append(fut)
log('submitted ')
check_futures()
log('loop ')
if workers:
worker_pool.shutdown()
check_futures()
def cppize_identifier(i: str) -> str: return common.fix_up_identification(common.shorten(i))
def run(quiet, start, end, workers):
global global_start, graphs_name_start, sparql_server, start_time, graphviz_pool
if start:
raise Exception("--start functionality needs updating")
global_start = start
if quiet:
logger.setLevel(logging.INFO)
sparql_server = sparql.SPARQLServer(sparql_uri)
redis_fn = redislite.Redis().db
info('redis is ' + redis_fn)
if workers:
worker_pool = ProcessPoolExecutor(max_workers=workers)
graphviz_pool = ProcessPoolExecutor(max_workers=128)
graphs_name_start = query_one('x', "{kbdbg:latest kbdbg:is ?x}")
identification0 = query_one(
'y', "{<" + graphs_name_start + "> kbdbg:has_run_identification ?y}")
path = 'runs/' + fix_up_identification(identification0)
info('output path:' + path)
os.system('mkdir -p ' + path)
identification = path + '/' + fix_up_identification(graphs_name_start)
graph_list_position = graphs_name_start
step_to_submit = -1
done = False
range_start = None
start_time = time.perf_counter()
range_size = 200000
while not done:
step_to_submit += 1
if step_to_submit < start - 1:
info("skipping [" + str(step_to_submit) + ']')
continue
if range_start == None:
range_start = step_to_submit
range_end = step_to_submit
if range_end - range_start == range_size or (range_end >= end
and end != -1):
args = (identification, 'step_graph_uri', range_start, range_end,
redis_fn)
if not workers:
work(*args)
else:
if check_futures() == 'end':
info("ending")
done = True
break
while len(futures) > workers + 1:
info('sleeping')
time.sleep(10)
if check_futures() == 'end':
info("ending")
done = True
break
info('submit ' + str(range_start) + '-' + str(range_end) +
' (queue size: ' + str(len(futures)) + ')')
fut = worker_pool.submit(work, *args)
fut.step = step_to_submit
futures.append(fut)
log('submitted ')
time.sleep(secs_per_frame)
if check_futures() == 'end':
info("ending")
done = True
break
range_start = range_end + 1
range_size = range_size * 5
if range_size >= 100000:
range_size = 100000
if range_start > end and end != -1:
info("ending")
done = True
break
log('loop' + str(step_to_submit))
if workers:
while len(futures) != 0:
check_futures()
info('waiting for workers to end')
time.sleep(10)
worker_pool.shutdown()
check_futures()
while len(graphviz_futures) != 0:
check_futures2(graphviz_futures)
info('waiting for graphviz workers to end')
time.sleep(2)
graphviz_pool.shutdown()
def set_new_identification():
global identification
identification = common.fix_up_identification(
fn + '_' + str(query_number if query_number != None else 0) +
additional_identification_)