def merge_files(self, merge_dir, file_type):
level = 0
merge_file_list = self.get_files_based_on_level(merge_dir, level)
count = len(merge_file_list)
pm = process_manager.ProcessManager()
pm.start_workers()
while count > 1:
while count > 1:
fh1 = merge_file_list[count - 1]
fh2 = merge_file_list[count - 2]
count = count - 2
if level == 0:
of = fh1 + ".level" + str(level + 1)
else:
of = fh1.replace(".level" + str(level),
".level" + str(level + 1))
mf = merge_files.MergeFiles(fh1, fh2, of, file_type)
pm.add_jobs(mf)
if count == 1:
# one single file left bump it up
self.bump_file_level(merge_file_list[count - 1], level)
# wait till processing is completed
while not pm.are_all_jobs_completed():
time.sleep(5)
level = level + 1
merge_file_list = self.get_files_based_on_level(merge_dir, level)
count = len(merge_file_list)
pm.close()
return merge_file_list[0]
def process(self):
self.open_files();
self.load_tld();
global_file.clean_dir(self.temp_dir);
# writing temp_file
group_counter = 0;
out_file_list = [];
pm = process_manager.ProcessManager(15);
pm.start_workers();
line = self.input_fh.readline();
while line != '':
counter = self.split_size;
temp_in_file_block = os.path.join(self.temp_dir,
str(group_counter)+".in");
temp_out_file_block = os.path.join(self.temp_dir,
str(group_counter) + ".out");
temp_in_fh = open(temp_in_file_block, "w");
while counter > 0 and line != '':
temp_in_fh.write(line);
line = self.input_fh.readline();
counter -= 1;
temp_in_fh.close();
dm = tld_geoip_mapping.TldGeoipMapping(temp_in_file_block,
temp_out_file_block,
self.tld);
pm.add_jobs(dm);
out_file_list.append(temp_out_file_block);
group_counter += 1;
# closing input_files
while not pm.are_all_jobs_completed():
time.sleep(5);
pm.close();
self.aggreate_results(out_file_list);
return;
def process(self):
self.open_files()
self.load_cctld()
global_file.clean_dir(self.temp_dir)
# writing temp_file
group_counter = 0
out_file_list = []
pm = process_manager.ProcessManager()
pm.start_workers()
line = self.input_fh.readline()
while line != '':
counter = self.split_size
temp_in_file_block = os.path.join(self.temp_dir,
str(group_counter) + ".in")
temp_out_file_block = os.path.join(self.temp_dir,
str(group_counter) + ".out")
temp_in_fh = open(temp_in_file_block, "w")
while counter > 0 and line != '':
temp_in_fh.write(line)
line = self.input_fh.readline()
counter -= 1
temp_in_fh.close()
dm = domain_mapper.DomainMapper(temp_in_file_block,
temp_out_file_block, self.cctld)
pm.add_jobs(dm)
out_file_list.append(temp_out_file_block)
group_counter += 1
# closing input_files
while not pm.are_all_jobs_completed():
time.sleep(5)
pm.close()
self.aggreate_results(out_file_list)
return
def process(self, wait=True):
pm = process_manager.ProcessManager()
file_list = self.get_source_files()
pm.start_workers()
for my_file in file_list:
be = basic_extraction.BasicExtraction(my_file, self.node_dir,
self.edge_dir)
pm.add_jobs(be)
if wait:
while not pm.are_all_jobs_completed():
time.sleep(5)
pm.close()
return
def main():
#sys.settrace(trace)
parser = setup_parser()
args = parser.parse_args()
entity = args.entity # get the entity from the argparser
streams = args.streams # the video streams to feed to the entity
cameras_list = args.cameras # get the cameras from the argparser
graphical = args.graphical # whether or not to show the streams
delay = args.delay # delay in between frames
single_process = args.single_process # whether to run it in a single process
if len(streams) < 1:
# user must provide at least one camera
parser.error("No camera stream given!")
if entity not in entities.entity_classes:
parser.error("'%s' is not a valid entity. Please check entities.entity_classes for valid names" % entity)
# convert the camera list to a dictionary format
cameras = {name: source for name, source in cameras_list}
if single_process:
class FakePipe(object):
def poll(self):
return None
def send(self, data):
pass #print data
process_manager.run_entity(
FakePipe(),
entities.entity_classes[entity],
*streams,
cameras=cameras,
delay=delay,
debug=graphical
)
else:
# spawn a process manager
pm = process_manager.ProcessManager(
extra_kwargs={
"cameras": cameras,
"delay": delay,
"debug": graphical,
}
)
# start the requested vision entity
pm.start_process(
entities.entity_classes[entity],
entity,
*streams
)
try:
while True:
output = pm.get_data()
# for debugging, print out entity output
if output and ENTITY_OUTPUT_PRINTS:
print output
except process_manager.KillSignal: # Exit if the subprocess tells us to
pass
except Exception:
pm.kill()
raise
import threading
import signal
import sys
import os
import hashlib
import argparse
import zmq
os.chdir(os.path.dirname(os.path.realpath(__file__)))
sys.dont_write_bytecode = True
if os.uname()[4] != "armv7l":
sys.path.insert(0, '../../lib')
import process_manager
processes = process_manager.ProcessManager()
processes.run_command("protoc -I. --proto_path=. " \
+ "--python_out=/tmp/ " \
+ "serial_comms_message.proto")
sys.path.insert(0, '/tmp')
import serial_comms_message_pb2 as serial_comms_message_proto
class GroundSerialComms:
def __init__(self, device):
self.ser = serial.Serial(
port=device, \
baudrate=57600, \
parity=serial.PARITY_NONE, \
stopbits=serial.STOPBITS_ONE, \
bytesize=serial.EIGHTBITS, \
def setUp(self):
self.test_drone = process_manager.ProcessManager()
signal.signal(signal.SIGINT, self.test_drone.killall_signalled)
def main():
''' Main que simula o dispatcher, administra todos os modulos e le os Arquivos
de entrada '''
#Inicializa modulos
manager = pm.ProcessManager()
memory = mm.MemoryManager()
io = iom.IOManager()
filesystem = fm.FileManager()
logger = log.Logger()
# Se tiver dois argumentos de linha, utiliza-os, senao usa os padroes
if len(sys.argv) > 2:
procFile = sys.argv[1]
arqFile = sys.argv[2]
else:
procFile = 'processes.txt'
arqFile = 'files.txt'
#Abre e le o arquivo de processos
with open(procFile, 'r') as f:
procs = [[int(x) for x in line.split(',')] for line in f]
processes = [pm.Process(x).__dict__ for x in procs]
#Abre e le o arquivo do sistema de arquivos
with open(arqFile, 'r') as f:
temp = f.read().splitlines()
filesystem.qtd_blocos = int(temp[0])
filesystem.qtd_segmentos = int(temp[1])
filesystem.arquivos = [
fm.File(temp[i].replace(' ', '').split(',')).__dict__
for i in range(2, filesystem.qtd_segmentos + 2)
]
# import ipdb; ipdb.set_trace()
filesystem.operacoes = [
fm.FileOperation(temp[i].replace(' ', '').split(',')).__dict__
for i in range(filesystem.qtd_segmentos + 2, len(temp))
]
filesystem.inicia_disco()
#Ordena os processos por ordem de chegada
manager.fila_principal = list(
sorted(processes, key=operator.itemgetter('tempo_init')))
# Como o quantum e um, o tratamento e apenas um iterador t
t = 0
while (True):
#Se tiver processo ainda nao processado
while (manager.fila_principal):
#Escalona processos do tempo de chegada = t
if (manager.fila_principal[0]['tempo_init'] == t):
manager.escalona_processo_geral()
else:
break
#Escalona processos da fila de usuario para as filas de prioridade
while (manager.escalona_processo_usuario()):
pass
#SE NAO TEM NADA EXECUTANDO(SE TIVER VAI SER TEMPO REAL)
if (not (manager.em_execucao)):
#Executa tempo real se tiver
for novo_processo in manager.fila_tempo_real:
#Tenta salvar na memoria, se tiver espaco
novo_processo['PID'] = manager.gera_pid()
offset = memory.salva(novo_processo)
#Coloca em execucao
if (offset is not None):
manager.em_execucao = manager.fila_tempo_real.pop(
manager.fila_tempo_real.index(novo_processo))
manager.em_execucao['offset'] = offset
logger.dispatch(manager.em_execucao)
break
#Nao atribui PID se n conseguir salvar na memoria
else:
novo_processo['PID'] = None
manager.ultimoPID -= 1
#Se nao tiver tempo real, vai ser despachado processos de usuario
else:
# Procura algum processo de prioridade 1 que possa ser executado
for novo_processo in manager.prioridade_1:
#Se processo ainda nao esta na memoria(nunca foi executado)
if novo_processo['offset'] is None:
#Ve se pode ser alocado em IO
novo_processo['PID'] = manager.gera_pid()
if (io.aloca(novo_processo)):
offset = memory.salva(novo_processo)
novo_processo['offset'] = offset
if offset is not None:
logger.dispatch(novo_processo)
offset = novo_processo['offset']
#Se o processo puder ser executado, carrega para a CPU
if (offset is not None):
manager.em_execucao = manager.prioridade_1.pop(
manager.prioridade_1.index(novo_processo))
break
else:
novo_processo['PID'] = None
manager.ultimoPID -= 1
# Se nao pode atribuir processos de prioridade 1(falta de processos ou recursos(memoria e io))
else:
for novo_processo in manager.prioridade_2:
#Se processo ainda nao esta na memoria
if novo_processo['offset'] is None:
#Ve se pode ser alocado em IO
novo_processo['PID'] = manager.gera_pid()
if (io.aloca(novo_processo)):
offset = memory.salva(novo_processo)
novo_processo['offset'] = offset
if offset is not None:
logger.dispatch(novo_processo)
offset = novo_processo['offset']
#Se o processo puder ser executado, carrega para a CPU
if (offset is not None):
manager.em_execucao = manager.prioridade_2.pop(
manager.prioridade_2.index(novo_processo))
break
else:
novo_processo['PID'] = None
manager.ultimoPID -= 1
# Se nao pode atribuir processos de prioridade 1 ou 2(falta de processos ou recursos(memoria e io))
else:
for novo_processo in manager.prioridade_3:
#Se processo ainda nao esta na memoria
if novo_processo['offset'] is None:
#Ve se pode ser alocado em IO
novo_processo['PID'] = manager.gera_pid()
if (io.aloca(novo_processo)):
offset = memory.salva(novo_processo)
novo_processo['offset'] = offset
if offset is not None:
logger.dispatch(novo_processo)
offset = novo_processo['offset']
#Se o processo puder ser executado, carrega para a CPU
if (offset is not None):
manager.em_execucao = manager.prioridade_3.pop(
manager.prioridade_3.index(novo_processo))
break
else:
novo_processo['PID'] = None
manager.ultimoPID -= 1
if (manager.acabou()):
#Condicao de saida do programa => Nao tem nenhum processo em nenhuma fila
#E todos os processos ja chegaram
break
# Executa Processo
if (manager.em_execucao):
#Decrementa tempo restante e aumenta o numero de instrucoes rodadas
manager.em_execucao['tempo_processador'] -= 1
manager.em_execucao['execucoes'] += 1
#Mostra Saida
logger.executa(manager.em_execucao)
#APOS EXECUCAO
#Remove o processo da memoria e libera recursos SE TIVER ACABADO O TEMPO
if manager.em_execucao['tempo_processador'] == 0:
filesystem.opera_processo(manager.em_execucao)
io.libera(manager.em_execucao)
memory.mata(manager.em_execucao)
manager.em_execucao = {}
#COmo o quantum eh um, processos de usuario sao retirados da CPU em toda iteracao
elif manager.em_execucao['prioridade'] > 0:
if manager.em_execucao['prioridade'] == 1:
manager.prioridade_1.append(manager.em_execucao)
elif manager.em_execucao['prioridade'] == 2:
manager.prioridade_2.append(manager.em_execucao)
elif manager.em_execucao['prioridade'] == 3:
manager.prioridade_3.append(manager.em_execucao)
manager.em_execucao = {}
#Avanca uma unidade de tempo
t += 1
#Mostra saida do sistema de arquivos
logger.disco(filesystem)