def _create_app(cls, app, number_of_tests):
app_path = os.path.join(cls.dir, app)
os.mkdir(app_path)
sub_call(
['django-admin.py', 'startapp', app, app_path]
)
cls._create_test_file(app_path, number_of_tests)
with open(os.path.join(cls.proj_path, 'settings.py'), 'a') as f:
f.write("INSTALLED_APPS += ('%s',)\n" % app)
def setUpClass(cls):
cls.patcher = patch('django_tdaemon.tdaemon.TEST', new=True)
cls.patcher.start()
cls.dir = create_tmp_dir()
sub_call(
['django-admin.py', 'startproject', 'test_proj',
cls.dir])
cls.proj_path = os.path.join(cls.dir, 'test_proj')
sys.path.append(cls.dir)
cls._create_app('app1', 1)
cls._create_app('app2', 2)
def main():
script_path = abspath(__file__)
script_dir = os_split(script_path)[0]
system('cd ' + script_dir)
dot_file_path = (script_dir + "\\" + json_extract('paths')['dot.exe'])
destination_file = json_extract('paths')['dot_file']
print(argv.__len__())
args = ""
if argv.__len__() < 2:
args = '-Tpng -o my_classes.png'
else:
for arg in argv[1:]:
args += arg
print(arg)
ar = [dot_file_path] + [destination_file] + split(args)
try:
sub_call(ar)
print('uml done')
except PermissionError:
print("you don't Hve valid permissions to create this file")
except FileNotFoundError:
print("cannot find file at " + ar)
def safe_copy(infile, outfile, sleep=10, attempts=10):
infile = infile.replace("@", "") if infile.startswith("@") else infile
# Try not to step on any toes....
if infile.startswith("root:"):
print 'file is on xrootd - switching to XRD library'
cmnd = "xrdcp %s %s" % (infile, outfile)
else:
cmnd = "cp %s %s" % (infile, outfile)
i = 0
logging.info("copy %s -> %s", infile, outfile)
while i < attempts:
status = sub_call(shlex.split(cmnd))
if status == 0:
return status
else:
logging.warning("%i - Copy failed; sleep %ss", i, sleep)
time.sleep(sleep)
i += 1
raise Exception("File *really* doesn't exist: %s" % infile)
def safe_copy(infile, outfile, sleep=10, attempts=10):
infile = infile.replace("@","") if infile.startswith("@") else infile
# Try not to step on any toes....
if infile.startswith("root:"):
print 'file is on xrootd - switching to XRD library'
cmnd = "xrdcp %s %s"%(infile,outfile)
else:
cmnd = "cp %s %s"%(infile,outfile)
i = 0
logging.info("copy %s -> %s",infile,outfile)
while i < attempts:
status = sub_call(shlex.split(cmnd))
if status == 0:
return status
else:
logging.warning("%i - Copy failed; sleep %ss", i,sleep)
time.sleep(sleep)
i += 1
raise Exception("File *really* doesn't exist: %s"%infile)
def clear_screen():
sub_call('cls', shell=True)
def _colordescriptor_generation(self, video_file, frame2file_map,
working_directory, output_file_csift,
output_file_tch):
"""
Generate colordescriptor output on the clip-level, generating
combined files to the given paths. These files wll have the format as
described by the ``combine_frame_results`` function (see below).
This method will generate the intermediary per-frame output files from
the colordescriptor executable as well as the two combined files for the
two colordescriptor modes to the paths specified.
:param video_file: The video to generated descriptor matrices from.
:type video_file: str
:param frame2file_map: dictionary mapping frame index to the image file
for that frame as was extracted from the given video file.
:type frame2file_map: dict of (int, str)
:param working_directory: The directory in which we will perform and
store our work.
:type working_directory: str
:param output_file_csift: The path to the file to output the combined
csift descriptor matrix.
:type output_file_csift: str
:param output_file_tch: The path to the file to output the combined tch
descriptor matrix.
:type output_file_tch: str
"""
# If either of the combined files exist, don't process that side's
# material.
# - This causes us to not even bother collecting potentially existing
# computed frame files.
# - This is for if one side completed but the other didn't due to some
# interruption or something.
compute_csift = not osp.isfile(output_file_csift)
compute_tch = not osp.isfile(output_file_tch)
if not (compute_csift or compute_tch):
# Quick exit when both are already computed.
self._log.info("Both CSIFT and TCH combined files have already "
"been computed.")
return
# NOTE:
# If we are continuing from this point on, then either both need a
# combined file computed, or just TCH, as its last to get interrupted.
# Thus, always check TCH frame computation and collect those files for
# TCH combination.
w, h, duration, fps = self.mp4_video_properties(video_file)
file_prefix, file_key = self.get_video_prefix(video_file)
# For pixel sample grid, we want to take at a maximum of 50,
# sample points in longest direction with at least a 6 pixel spacing. We
# will take fewer sample points to ensure the 6 pixel minimum spacing.
# (magic numbers are
# a result of tuning)
sample_size = max(int(math.floor(max(w, h) / 50.0)), 6)
total_frames = len(frame2file_map)
# Output files used by this method. These files will also act like stamp
# files, detailing progress from previous runs. Files will be removed
# from the disk when the final product of this method has been
# completed. When this file is present before processing occurs, a total
# processing skip occurs.
cd_output_file_pattern = osp.join(working_directory, "%s-%06d.txt")
cd_log_file_pattern = osp.join(working_directory, "%s-%06d.log")
# Files will be added to these maps keyed by their frame/index number
csift_frame_feature_files = {}
tch_frame_feature_files = {}
def construct_command(descriptor, input_frame_file, output_file):
return (self.cdescriptor_exe, input_frame_file, '--detector',
'densesampling', '--ds_spacing', str(sample_size),
'--descriptor', descriptor, '--output', output_file)
self._log.info(
"starting per-frame descriptor generation "
"(%d frame files)", len(frame2file_map))
dev_null = open('/dev/null', 'w')
for i, (frame, png_frame) in enumerate(frame2file_map.items()):
# For the two processing components, if the predicted generated file
# already exists, skip processing for this frame. Else, remove old
# tmp file, generated new tmp file, rename to legitimate name.
# CSIFT execution
if compute_csift:
csift_output_file = cd_output_file_pattern % ('csift', frame)
if not osp.isfile(csift_output_file):
csift_log_file = cd_log_file_pattern % ('csift', frame)
csift_tmp_file = self.tempify_filename(csift_output_file)
if osp.isfile(csift_tmp_file):
# remove the file if it exists from previous interrupted
# processing
os.remove(csift_tmp_file)
self._log.info(
"[vID:{vid:s} - frm:{cur:d}/{tot:d} ({perc:8.3%})] "
"processing csift features".format(
vid=file_key,
cur=i,
tot=total_frames - 1,
perc=float(i * 2) / (total_frames * 2)))
self._log.debug(
'call: %s', ' '.join(
construct_command('csift', png_frame,
csift_tmp_file)))
with open(csift_log_file, 'w') as lfile:
ret = sub_call(construct_command(
'csift', png_frame, csift_tmp_file),
stdout=lfile,
stderr=lfile)
if ret != 0:
raise RuntimeError("Failed to process colordescriptor "
"csift on frame %s for video %s." %
(png_frame, video_file))
try:
os.rename(csift_tmp_file, csift_output_file)
except:
print "Trying to rename: " + csift_tmp_file
else:
self._log.info(
"[vID:{vid:s} - frm:{cur:d}/{tot:d} ({perc:8.3%})] "
"csift features already processed".format(
vid=file_key,
cur=i,
tot=total_frames - 1,
perc=float(i * 2) / (total_frames * 2)))
csift_frame_feature_files[frame] = csift_output_file
# TCH execution
tch_output_file = cd_output_file_pattern % ('tch', frame)
if not osp.isfile(tch_output_file):
tch_log_file = cd_log_file_pattern % ('tch', frame)
tch_tmp_file = self.tempify_filename(tch_output_file)
if osp.isfile(tch_tmp_file):
os.remove(tch_tmp_file)
self._log.info(
"[vID:{vid:s} - frm:{cur:d}/{tot:d} ({perc:8.3%})] "
"processing tch features".format(vid=file_key,
cur=i,
tot=total_frames - 1,
perc=float(i * 2 + 1) /
(total_frames * 2)))
self._log.debug(
'call: %s', ' '.join(
construct_command('transformedcolorhistogram',
png_frame, tch_tmp_file)))
with open(tch_log_file, 'w') as lfile:
ret = sub_call(construct_command(
'transformedcolorhistogram', png_frame, tch_tmp_file),
stdout=lfile,
stderr=lfile)
if ret != 0:
raise RuntimeError("Failed to process colordescriptor tch "
"on frame %s for video %s." %
(png_frame, video_file))
try:
os.rename(tch_tmp_file, tch_output_file)
except:
pass
else:
self._log.info(
"[vID:{vid:s} - frm:{cur:d}/{tot:d} ({perc:8.3%})] "
"tch features already processed".format(
vid=file_key,
cur=i,
tot=total_frames - 1,
perc=float(i * 2 + 1) / (total_frames * 2)))
tch_frame_feature_files[frame] = tch_output_file
# combineResults (local method)
if compute_csift:
self._log.debug('combining csift feature matrices -> %s',
output_file_csift)
tmp_file = self.tempify_filename(output_file_csift)
self._combine_frame_results(csift_frame_feature_files, tmp_file)
os.rename(tmp_file, output_file_csift)
self._log.debug('combining tch feature matrices -> %s',
output_file_tch)
tmp_file = self.tempify_filename(output_file_tch)
self._combine_frame_results(tch_frame_feature_files, tmp_file)
os.rename(tmp_file, output_file_tch)
def call(cmd_string):
print 'CALLING: %s' % cmd_string
r = sub_call(cmd_string,shell=True)
def _colordescriptor_generation(self, video_file, frame2file_map,
working_directory,
output_file_csift, output_file_tch):
"""
Generate colordescriptor output on the clip-level, generating
combined files to the given paths. These files wll have the format as
described by the ``combine_frame_results`` function (see below).
This method will generate the intermediary per-frame output files from
the colordescriptor executable as well as the two combined files for the
two colordescriptor modes to the paths specified.
:param video_file: The video to generated descriptor matrices from.
:type video_file: str
:param frame2file_map: dictionary mapping frame index to the image file
for that frame as was extracted from the given video file.
:type frame2file_map: dict of (int, str)
:param working_directory: The directory in which we will perform and
store our work.
:type working_directory: str
:param output_file_csift: The path to the file to output the combined
csift descriptor matrix.
:type output_file_csift: str
:param output_file_tch: The path to the file to output the combined tch
descriptor matrix.
:type output_file_tch: str
"""
# If either of the combined files exist, don't process that side's
# material.
# - This causes us to not even bother collecting potentially existing
# computed frame files.
# - This is for if one side completed but the other didn't due to some
# interruption or something.
compute_csift = not osp.isfile(output_file_csift)
compute_tch = not osp.isfile(output_file_tch)
if not (compute_csift or compute_tch):
# Quick exit when both are already computed.
self._log.info("Both CSIFT and TCH combined files have already "
"been computed.")
return
# NOTE:
# If we are continuing from this point on, then either both need a
# combined file computed, or just TCH, as its last to get interrupted.
# Thus, always check TCH frame computation and collect those files for
# TCH combination.
w, h, duration, fps = self.mp4_video_properties(video_file)
file_prefix, file_key = self.get_video_prefix(video_file)
# For pixel sample grid, we want to take at a maximum of 50,
# sample points in longest direction with at least a 6 pixel spacing. We
# will take fewer sample points to ensure the 6 pixel minimum spacing.
# (magic numbers are
# a result of tuning)
sample_size = max(int(math.floor(max(w, h) / 50.0)), 6)
total_frames = len(frame2file_map)
# Output files used by this method. These files will also act like stamp
# files, detailing progress from previous runs. Files will be removed
# from the disk when the final product of this method has been
# completed. When this file is present before processing occurs, a total
# processing skip occurs.
cd_output_file_pattern = osp.join(working_directory, "%s-%06d.txt")
cd_log_file_pattern = osp.join(working_directory, "%s-%06d.log")
# Files will be added to these maps keyed by their frame/index number
csift_frame_feature_files = {}
tch_frame_feature_files = {}
def construct_command(descriptor, input_frame_file, output_file):
return (self.cdescriptor_exe, input_frame_file,
'--detector', 'densesampling',
'--ds_spacing', str(sample_size),
'--descriptor', descriptor,
'--output', output_file)
self._log.info("starting per-frame descriptor generation "
"(%d frame files)", len(frame2file_map))
dev_null = open('/dev/null', 'w')
for i, (frame, png_frame) in enumerate(frame2file_map.items()):
# For the two processing components, if the predicted generated file
# already exists, skip processing for this frame. Else, remove old
# tmp file, generated new tmp file, rename to legitimate name.
# CSIFT execution
if compute_csift:
csift_output_file = cd_output_file_pattern % ('csift', frame)
if not osp.isfile(csift_output_file):
csift_log_file = cd_log_file_pattern % ('csift', frame)
csift_tmp_file = self.tempify_filename(csift_output_file)
if osp.isfile(csift_tmp_file):
# remove the file if it exists from previous interrupted
# processing
os.remove(csift_tmp_file)
self._log.info(
"[vID:{vid:s} - frm:{cur:d}/{tot:d} ({perc:8.3%})] "
"processing csift features"
.format(vid=file_key, cur=i,
tot=total_frames - 1,
perc=float(i * 2) / (total_frames * 2))
)
self._log.debug('call: %s',
' '.join(construct_command('csift', png_frame,
csift_tmp_file)))
with open(csift_log_file, 'w') as lfile:
ret = sub_call(construct_command('csift', png_frame,
csift_tmp_file),
stdout=lfile, stderr=lfile)
if ret != 0:
raise RuntimeError("Failed to process colordescriptor "
"csift on frame %s for video %s."
% (png_frame, video_file))
try:
os.rename(csift_tmp_file, csift_output_file)
except:
print "Trying to rename: " + csift_tmp_file
else:
self._log.info(
"[vID:{vid:s} - frm:{cur:d}/{tot:d} ({perc:8.3%})] "
"csift features already processed"
.format(vid=file_key, cur=i,
tot=total_frames - 1,
perc=float(i * 2) / (total_frames * 2))
)
csift_frame_feature_files[frame] = csift_output_file
# TCH execution
tch_output_file = cd_output_file_pattern % ('tch', frame)
if not osp.isfile(tch_output_file):
tch_log_file = cd_log_file_pattern %('tch', frame)
tch_tmp_file = self.tempify_filename(tch_output_file)
if osp.isfile(tch_tmp_file):
os.remove(tch_tmp_file)
self._log.info(
"[vID:{vid:s} - frm:{cur:d}/{tot:d} ({perc:8.3%})] "
"processing tch features"
.format(vid=file_key, cur=i,
tot=total_frames - 1,
perc=float(i * 2 + 1) / (total_frames * 2))
)
self._log.debug(
'call: %s',
' '.join(construct_command('transformedcolorhistogram',
png_frame, tch_tmp_file))
)
with open(tch_log_file, 'w') as lfile:
ret = sub_call(construct_command('transformedcolorhistogram',
png_frame, tch_tmp_file),
stdout=lfile, stderr=lfile)
if ret != 0:
raise RuntimeError("Failed to process colordescriptor tch "
"on frame %s for video %s."
% (png_frame, video_file))
try:
os.rename(tch_tmp_file, tch_output_file)
except:
pass
else:
self._log.info(
"[vID:{vid:s} - frm:{cur:d}/{tot:d} ({perc:8.3%})] "
"tch features already processed"
.format(vid=file_key, cur=i,
tot=total_frames - 1,
perc=float(i * 2 + 1) / (total_frames * 2))
)
tch_frame_feature_files[frame] = tch_output_file
# combineResults (local method)
if compute_csift:
self._log.debug('combining csift feature matrices -> %s',
output_file_csift)
tmp_file = self.tempify_filename(output_file_csift)
self._combine_frame_results(csift_frame_feature_files, tmp_file)
os.rename(tmp_file, output_file_csift)
self._log.debug('combining tch feature matrices -> %s',
output_file_tch)
tmp_file = self.tempify_filename(output_file_tch)
self._combine_frame_results(tch_frame_feature_files, tmp_file)
os.rename(tmp_file, output_file_tch)
def play_url(self, url):
cmd = [__cmd] + __args
if not __vid:
cmd += ["--no-video"]
cmd += url
sub_call(cmd)