def __init__(self, opt):
self.file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
opt.file)
process = Process(self.file, opt)
self.inputs, self.labels, _ = process.get_data()
# feature length
self.p = process.feature_length
self.k = opt.k # 参数v 的维度
self.lr = opt.lr # 学习率
self.reg_l1 = opt.reg_l1
self.reg_l2 = opt.reg_l2
self.data_length = process.data_length # 数据总量
self.opt = opt
self.train_log = os.path.join(
os.path.dirname(os.path.abspath(__file__)), opt.train_log)
self.save_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), opt.save_path)
save_parent_path = os.path.split(self.save_path)[0]
if not os.path.exists(self.train_log):
os.makedirs(self.train_log)
if not os.path.exists(save_parent_path):
os.makedirs(save_parent_path)
def test_large_output():
# There used to be a bug in subprocess.PIPE that causes subprocess.Process
# to hang indefinitely if the child process produces more than or equal
# to 2^16 bytes of data
# https://thraxil.org/users/anders/posts/2008/03/13/Subprocess-Hanging-PIPE-is-your-enemy/
process = Process(
(sys.executable, "-c", "import time; print('x ' * (2 ** 16))"),
print_stdout=True)
print(process.run())
def __init__(self, opt):
self.file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
opt.file)
process = Process(self.file, opt)
self.inputs, self.labels, _ = process.get_data()
self.p = process.feature_length
if opt.if_all_feature:
self.feature2field = process.feature2field # ind 对应的域
self.p1 = self.p
else:
#就取前几维特征做测试
self.p1 = opt.feature_num
self.feature2field = {}
feature_ind = 0
for key, value in process.feature2field.items():
if feature_ind > self.p1:
break
self.feature2field[key] = value
feature_ind += 1
self.k = opt.k # 参数v 的维度
self.lr = opt.lr # 学习率
self.reg_l1 = opt.reg_l1
self.reg_l2 = opt.reg_l2
self.data_length = process.data_length # 数据总量
self.f = len(
process.fields_dict.keys()) - 2 #除去all_count feature2field
self.opt = opt
self.train_log = os.path.join(
os.path.dirname(os.path.abspath(__file__)), opt.train_log)
self.save_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), opt.save_path)
save_parent_path = os.path.split(self.save_path)[0]
if not os.path.exists(self.train_log):
os.makedirs(self.train_log)
if not os.path.exists(save_parent_path):
os.makedirs(save_parent_path)
def compileFile(self, name):
"""
Compiles the given file.
Parameters:
----------
name:
The name of the file to compile.
Returns:
-------
The stdout, stderr, and return code of the compiler.
"""
compileProc = Process()
compileProc.procName = self.compiler
compileProc.procArgs = [name]
compileOut, compileErr, compileCode = compileProc.runPiped()
compileOut = self.convertByteString(compileOut)
compileErr = self.convertByteString(compileErr)
return compileCode, compileErr, compileOut
def runFile(self, name):
"""
Runs the program after being compiled.
This will also capture stdout, stderr, and use any input files as
stdin.
Parameters:
----------
name:
The name of the file to run.
Returns:
-------
The stdout, stderr, and return code of the program.
"""
runProc = Process()
runProc.procName = self.run
runProc.procArgs = [name]
# Check if there is an input file that needs to be used.
inputFile = ''
for file in self.inputFiles:
fName = os.path.splitext(basename(file))[0]
if fName == name:
inputFile = file
break
if inputFile:
with open(inputFile, 'r') as inFile:
inLines = inFile.read()
inLines = str.encode(inLines)
runOut, runErr, runCode = runProc.runPiped(input=inLines)
else:
runOut, runErr, runCode = runProc.runPiped()
runOut = self.convertByteString(runOut)
runErr = self.convertByteString(runErr)
return runCode, runErr, runOut
def cpu_scheduler(p_table, algorithm):
"""
:param p_table: a list of processes - instances of class Process.
:param algorithm: an integer in range [1,8].
:return: next process that must be dispatched on cpu.
"""
current_time = 0
current_p_table = []
enter_times = [99999]
preemptive = (5, 6, 7, 8)
non_preemptive = (1, 2, 3, 4)
real_time = (7, 8)
for process in p_table:
if process.enter not in enter_times:
enter_times.append(process.enter)
enter_times.sort(reverse=True)
next_entrance = enter_times.pop()
while len(p_table) is not 0:
for process in p_table:
if process not in current_p_table:
if process.enter <= current_time:
current_p_table.append(process)
next_process = find_next_process(p_table, algorithm, current_time)
print('{} : process {} is running'.format(current_time,
next_process.name))
if algorithm in real_time:
try:
time.sleep(1)
except KeyboardInterrupt:
print('\nKeyboardInterrupt.')
break
if algorithm in non_preemptive:
p_table.pop(p_table.index(next_process))
current_time += next_process.burst
print('{} : ...terminated'.format(current_time))
elif algorithm in preemptive:
if current_time >= next_entrance:
next_entrance = enter_times.pop()
burst_time = (next_entrance - current_time)
if burst_time < next_process.burst:
next_process.burst -= burst_time
current_time = next_entrance
else:
process = p_table.pop(p_table.index(next_process))
if algorithm in real_time:
new_process = Process(process.name, process.enter,
process.service, process.burst)
new_process.enter += process.service
p_table.append(new_process)
if new_process.enter not in enter_times:
enter_times.append(new_process.enter)
enter_times.sort(reverse=True)
current_time += next_process.burst
print('{} : ...terminated'.format(current_time))
else:
raise ValueError('The entered value is not in defined range')
print('\n')
return
def test2():
process = Process(
(sys.executable, "-c", "import time\nfor i in range(10):\n\t"
"time.sleep(0.3)\n\tprint(i)"),
print_stdout=True)
print(process.run())
def mark(self, rootDir, rubric):
"""
This is the main function of the Marker.
This will iterate over the directory of each student, read their
submission, compile and run it. It will then capture their output and
diff it. This will then be sent to the editor so the TA can mark the
assignment. It can also restore the list using an incremental file.
Parameters:
----------
rootDir:
The root of the assignemnts.
rubric:
The marking rubric to use.
Returns:
-------
The table containing all of the students, their marks and comments.
"""
table = []
# Check if we have a partial file already.
incPath = os.path.join(self.workingDir, 'grades_inc.csv')
incFile = Path(incPath)
start = 0
if incFile.is_file():
table, start = self.loadIncremental(incPath, rubric)
# Next, check copy over any input and output files to the working
# directory.
count = 0
for entry in os.scandir(rootDir):
if not entry.is_dir():
continue
if start is not 0:
start -= 1
continue
name = entry.name
subPath = os.path.join(entry.path, 'Submission attachment(s)')
submission = [
file for file in os.scandir(subPath) if file.is_file()
]
bundle = [submission]
for file in self.inputFiles:
shutil.copy2(file, self.workingDir)
for file in self.outputFiles:
shutil.copy2(file, self.workingDir)
for file in self.auxFiles:
shutil.copy2(file, self.workingDir)
if self.preProcessScript:
shutil.copy2(self.preProcessScript, self.workingDir)
# Now copy the submission over to the working directory.
for file in submission:
shutil.copy2(file.path, self.workingDir)
os.chdir(self.workingDir)
# Check if we have to run anything before.
if self.preProcessScript:
proc = Process()
proc.procName = 'python'
proc.procArgs = [self.preProcessScript]
proc.run()
try:
list = self.runSubmission(bundle)
except Exception as e:
print('Error in entry {}'.format(count))
print('Path: {}'.format(subPath))
print(traceback.format_exc())
self.writeIncremental(table, rubric)
continue
with open('rubric.txt', 'w+') as rubricFile:
i = 0
for item, mark in rubric.attributes.items():
rubricFile.write('{}: {}/{}\n'.format(
item, mark, rubric.maxVals[i]))
i += 1
rubricFile.write('#==============================#\n')
rubricFile.write('# Instructor comments\n')
rubricFile.write('#==============================#\n')
rubricFile.write('')
list.append('rubric.txt')
self.editor.run(list)
# The grader has now entered the grades and comments, so lets
# re-open the file and update the marks.
studentRubric = Rubric()
studentRubric.make(rubric)
studentRubric.studentName = name
with open('rubric.txt', 'r+') as rubricFile:
header = 0
comments = []
for line in rubricFile:
if line.startswith('#'):
header += 1
continue
if header is 3:
comments.append(line)
continue
tokens = line.split(':')
item = tokens[0]
vals = tokens[1].split('/')
studentRubric.attributes[item] = float(vals[0])
comments = ' '.join(comments)
studentRubric.comments = comments
studentRubric.addMarks()
table.append(studentRubric)
self.writeIncremental(table, rubric)
try:
os.remove('rubric.txt')
except:
pass
try:
os.remove('summary.txt')
except:
pass
for file in list:
if file == 'rubric.txt' or file == 'summary.txt':
continue
os.remove(file)
# Now remove any generated files.
for file in os.scandir(self.workingDir):
if not file.is_file():
continue
if self.generatedExtension not in file.name:
continue
os.remove(file.path)
print('Done')
return table
def process_video(
model,
thre,
input_path,
output_path,
require_fps,
hat_color,
person_color,
fourcc='avc1',
step=2,
):
"""处理视频并输出到指定目录
Arguments:
model {torch.nn.Sequ} -- [使用的模型]
input_path {[str]} -- [视频文件路径]
require_fps {[int]} -- [输出的视频fps]
fourcc {[str]} -- [opencv写文件编码格式]
hat_color {[str]} -- [安全帽框颜色]
person_color {[str]} -- [人头框颜色]
"""
video = mmcv.VideoReader(input_path,cache_capacity=40)
# 图像分辨率
resolution = (video.width, video.height)
video_fps = video.fps
ds = DetectionSifter(
int(video_fps),
osp.basename(args.i).split('.')[0],
resolution,
get_collection(asyn=True)
)
if require_fps is None:
require_fps = video_fps
if require_fps > video_fps:
require_fps = video_fps
vwriter = cv2.VideoWriter(
output_path,
VideoWriter_fourcc(*fourcc),
require_fps,
resolution
)
t = threading.Thread(target=write_frame,args=(vwriter,))
t.start()
# 跳步视频（每次读取step+1帧）
#video = over_step_video(video,step)
vlen = len(video)
indexs = np.arange(vlen)[0:len(video):step]
# 每次取step张图像，比如第一次取[0:3]第二次则取[2:5]确保探测一张跳过step-1张,相当于探测一张相当于探测step张
p = Process(model,step+1)
for start_index in tqdm(indexs):
end_index = start_index + step + 1
if end_index >= vlen:
break
origin_frames = video[start_index:end_index]
origin_frames = np.array(origin_frames)
frames_index = [start_index,start_index+step]
origin_frames,psn_objects,hat_objects = p.get_img(origin_frames,frames_index)
for psn_o in psn_objects:
ds.add_object(*psn_o)
Img_Q.put(origin_frames[:-1])
ds.clear()
print('process finshed')