def open_file_browser(self, dialog_ui: Optional[Ui_Dialog] = None):
try:
if not dialog_ui:
return
# Get the starting path
start_path = dialog_ui.pathTextEdit.toPlainText()
if start_path:
start_path, _ = ntpath.split(start_path)
start_path += os.path.sep + "*"
else:
start_path = None
# Open the file browser and add the selected path to the path text field
new_path = easygui.fileopenbox(title="Remote Play Anything",
msg="Choose the game executable",
default=start_path,
filetypes=["*.exe"])
if not new_path:
return
# If no name is in the name field or the name field is unchanged, set it as the file name
curr_name = dialog_ui.nameTextEdit.toPlainText()
curr_path = dialog_ui.pathTextEdit.toPlainText()
curr_file_name = utils.get_file_name(curr_path)
if not curr_name or curr_file_name == curr_name:
new_name = utils.get_file_name(new_path)
dialog_ui.nameTextEdit.setPlainText(new_name)
# Then set the path field
dialog_ui.pathTextEdit.setPlainText(new_path)
except Exception as e:
utils.show_error(text=str(e))
def get_test_cases(exclude=[]):
all_tests = [t for t in os.listdir('.') if test_name_pattern.match(utils.get_file_name(t))]
tcs = list()
for t in [t for t in all_tests if utils.get_file_name(t) not in exclude]:
with open(t, 'r') as inp:
content = inp.read()
tcs.append(utils.TestCase(utils.get_file_name(t), t, content))
return tcs
def get_test_cases(exclude=[], directory=utils.tmp):
all_tests = [t for t in glob.glob(directory + '/vector[0-9]*.test')]
tcs = list()
for t in [t for t in all_tests if utils.get_file_name(t) not in exclude]:
with open(t, 'r') as inp:
content = inp.read()
tcs.append(utils.TestCase(utils.get_file_name(t), t, content))
return tcs
def __getitem__(self, index):
try:
lr_file = self.lr_list[index]
hr_file = self.hr_list[index]
interval = random.choice(self.interval_list)
lr_path,lr_name = util.get_file_name(lr_file)
hr_path, _ = util.get_file_name(hr_file)
center_frame_idx = int(lr_name)
neighbor_list = getNeighbor(center_frame_idx,self.n_frames,interval)
if self.random_reverse and random.random() < 0.5:
neighbor_list.reverse()
file_text = '{:03d}.png'.format(center_frame_idx)
#### get lr images
lr_data_list = []
for v in neighbor_list:
lr_data_path = osp.join(lr_path, '{:03d}.png'.format(v))
lr_data_list.append(read_img(lr_data_path))
#### get hr images
hr_data_path = osp.join(hr_path, file_text)
hr_data = read_img(hr_data_path)
# randomly crop
height, width, channel = hr_data.shape
hr_size_w,hr_size_h = self.patch_size * self.scale, self.patch_size * self.scale
lr_height = height // self.scale
lr_width = width // self.scale
rnd_h = random.randint(0, max(0, lr_height - self.patch_size))
rnd_w = random.randint(0, max(0, lr_width - self.patch_size))
img_lr_list = [one_data[rnd_h:rnd_h + self.patch_size, rnd_w:rnd_w + self.patch_size, :] for one_data in lr_data_list]
rnd_h_hr, rnd_w_hr = int(rnd_h * self.scale), int(rnd_w * self.scale)
img_hr = hr_data[rnd_h_hr:rnd_h_hr + hr_size_h, rnd_w_hr:rnd_w_hr + hr_size_w, :]
# augmentation - flip, rotate
img_lr_list.append(img_hr)
rlt = util.augment(img_lr_list, hflip=True, rot=True)
img_lr_list = rlt[0:-1]
img_hr = rlt[-1]
# stack lr images to NHWC, N is the frame number
img_lrs = np.stack(img_lr_list, axis=0)
# HWC to CHW, numpy to tensor
img_hr = torch.from_numpy(np.ascontiguousarray(np.transpose(img_hr, (2, 0, 1)))).float()
# img_lrs = [torch.from_numpy(np.ascontiguousarray(np.transpose(img, (2, 0, 1)))).float() for img in img_lr_list]
img_lrs = torch.from_numpy(np.ascontiguousarray(np.transpose(img_lrs, (0, 3, 1, 2)))).float()
except Exception as e:
random_sum = random.randrange(0, self.__len__())
return self.__getitem__(random_sum)
return {'LRs': img_lrs, 'HR': img_hr}
def evaluate(model, dataloader, epoch, writer, logger, data_name='val'):
save_root = os.path.join(opt.result_dir, opt.tag, str(epoch), data_name)
utils.try_make_dir(save_root)
total_psnr = 0.0
total_ssim = 0.0
ct_num = 0
# print('Start testing ' + tag + '...')
for i, sample in enumerate(dataloader):
utils.progress_bar(i, len(dataloader), 'Eva... ')
path = sample['path']
with torch.no_grad():
recovered = model(sample)
if data_name == 'val':
label = sample['label']
label = tensor2im(label)
recovered = tensor2im(recovered)
ct_num += 1
total_psnr += psnr(recovered, label, data_range=255)
total_ssim += ski_ssim(recovered,
label,
data_range=255,
multichannel=True)
save_dst = os.path.join(save_root,
utils.get_file_name(path[0]) + '.png')
Image.fromarray(recovered).save(save_dst)
elif data_name == 'test':
pass
else:
raise Exception('Unknown dataset name: %s.' % data_name)
# 保存结果
save_dst = os.path.join(save_root,
utils.get_file_name(path[0]) + '.png')
Image.fromarray(recovered).save(save_dst)
if data_name == 'val':
ave_psnr = total_psnr / float(ct_num)
ave_ssim = total_ssim / float(ct_num)
# write_loss(writer, f'val/{data_name}', 'psnr', total_psnr / float(ct_num), epochs)
logger.info(f'Eva({data_name}) epoch {epoch}, psnr: {ave_psnr}.')
logger.info(f'Eva({data_name}) epoch {epoch}, ssim: {ave_ssim}.')
return f'{ave_ssim: .3f}'
else:
return ''
def __init__(self, data, model, args):
self.data = data
self.model = model
self.args = args
if isinstance(model, BrainAgeModel1):
self.file_name = utils.get_file_name('model1', 'csv', args)
elif isinstance(model, BrainAgeModel2):
self.file_name = utils.get_file_name('model2', 'csv', args)
else:
raise Exception('Invalid model')
def get_test_cases(exclude=[], directory=tests_dir):
all_tests = [t for t in glob.glob(directory + '/*.ktest')]
tcs = list()
for t in [t for t in all_tests if utils.get_file_name(t) not in exclude]:
print(t)
file_name = utils.get_file_name(t)
with open(t, mode='rb') as inp:
content = inp.read()
tcs.append(utils.TestCase(file_name, t, content))
return tcs
def get_test_cases(exclude=[], directory=tests_dir):
all_tests = [t for t in glob.glob(directory + '/*.ktest')]
tcs = list()
for t in [t for t in all_tests if utils.get_file_name(t) not in exclude]:
print(t)
file_name = utils.get_file_name(t)
with open(t, mode='rb') as inp:
content = inp.read()
tcs.append(utils.TestCase(file_name, t, content))
return tcs
def get_test_cases(exclude=[]):
all_tests = [
t for t in os.listdir('.')
if test_name_pattern.match(utils.get_file_name(t))
]
tcs = list()
for t in [t for t in all_tests if utils.get_file_name(t) not in exclude]:
with open(t, 'r') as inp:
content = inp.read()
tcs.append(utils.TestCase(utils.get_file_name(t), t, content))
return tcs
def download_agent(self):
if self.need_download:
if self.download_path and self.checksum:
url = '%s/%s' % (self.configs['update_server_url'], self.download_path)
f = urllib2.urlopen(url)
package_name = utils.get_file_name(url)
package_path = '%s/%s' % (self.get_download_dir(), package_name)
with open(package_path, 'wb') as target:
target.write(f.read())
if not utils.check_file(package_path, self.checksum):
os.remove(package_path)
LOG.debug('checksum failed, removing %s', package_name)
return
else:
LOG.debug('download %s success', package_name)
# src_url = '%s/agent_linux_2.tar.gz' % self.get_tmp_dir()
# log = os.popen('mv %s %s/' % (src_url, self.get_download_dir()))
# LOG.debug(log)
self.need_download = False
self.download_path = None
self.checksum = None
self.need_update = False
else:
self.need_update = False
return self.get_latest_install_package()
def scan():
tic = time.time()
if 'image' in request.files:
image_fs = request.files['image']
image_file = get_file_name(image_fs.filename)
try:
image_fs.save(image_file)
img = open_image(image_file)
rt = time.time() - tic
_, _, preds = learn.predict(img)
tt = time.time() - tic
ret_val = list(
map(
lambda x: {
'class': x[0],
'confidence': f"{x[1]:.2f}"
},
sorted(zip(classes, preds),
key=lambda x: x[1],
reverse=True)))
size = os.stat(image_file).st_size
debug_log(image_file, size, rt, tt)
return jsonify(ret_val)
except Exception as e:
logging.error(e)
return jsonify({"error": "Internal server error"}), 500
else:
return jsonify({"error": "data is required"}), 400
def train_ffn_model(x_train, y_train, x_val, y_val, params):
fnn_model = model.FFNModel(input_size=x_train.shape[1],
output_size=params["output_size"],
model_name=params["model_name"],
hidden_activation=params["hidden_activation"],
out_activation=params["out_activation"],
hidden_dims=params["hidden_dims"],
layers=params["layers"],
kernel_initializer=params["kernel_initializer"],
kernel_regularizer=params["kernel_regularizer"],
dropouts=params["dropouts"])
history = train_model(fnn_model,
x_train,
y_train,
params["out_dir"],
validation_data=(x_val, y_val),
save_checkpoint=params["save_checkpoint"],
n_epochs=params["n_epochs"],
batch_size=params["batch_size"],
verbose=params["verbose"],
early_stopping=params["early_stopping"],
learning_rate=params["learning_rate"],
loss=params["loss"],
ckpt_name_prefix=utils.get_file_name(params))
return utils.extract_results_from_history(history)
def main():
parser = argparse.ArgumentParser(
prog="mipsa",
description=
'Assemble a MIPS assembly program. Outputs an object file for every input file.'
)
parser.add_argument("files",
action="store",
nargs="+",
type=str,
help="list of assembly files to process")
parser.add_argument("--int",
action="store_true",
default=False,
help="output intermediate files")
args = parser.parse_args()
for input_file in args.files:
ints, objs = assemble(input_file)
file_name = utils.get_file_name(input_file)
if args.int:
int_file = file_name + ".int"
utils.write_file_from_list(int_file, ints)
obj_file = file_name + ".o"
utils.write_file_from_list(obj_file, objs)
def resize(image_path,
xml_path,
newSize,
output_path,
save_box_images=False,
verbose=False):
image = cv2.imread(image_path)
if image.shape[0] > image.shape[1]:
image = rotate_image(image)
scale_x = newSize[0] / image.shape[1]
scale_y = newSize[1] / image.shape[0]
image = cv2.resize(image, (newSize[0], newSize[1]))
newBoxes = []
xmlRoot = ET.parse(xml_path).getroot()
xmlRoot.find('filename').text = image_path.split('/')[-1]
size_node = xmlRoot.find('size')
size_node.find('width').text = str(newSize[0])
size_node.find('height').text = str(newSize[1])
for member in xmlRoot.findall('object'):
bndbox = member.find('bndbox')
xmin = bndbox.find('xmin')
ymin = bndbox.find('ymin')
xmax = bndbox.find('xmax')
ymax = bndbox.find('ymax')
xmin.text = str(np.round(int(float(xmin.text)) * scale_x))
ymin.text = str(np.round(int(float(ymin.text)) * scale_y))
xmax.text = str(np.round(int(float(xmax.text)) * scale_x))
ymax.text = str(np.round(int(float(ymax.text)) * scale_y))
newBoxes.append([
1, 0,
int(float(xmin.text)),
int(float(ymin.text)),
int(float(xmax.text)),
int(float(ymax.text))
])
if not checkmaxcoordinateslabels(newSize[0], int(float(xmin.text))):
continue
if not checkmaxcoordinateslabels(newSize[1], int(float(ymin.text))):
continue
if not checkmaxcoordinateslabels(newSize[0], int(float(xmax.text))):
continue
if not checkmaxcoordinateslabels(newSize[1], int(float(ymax.text))):
continue
(_, file_name, ext) = get_file_name(image_path)
cv2.imwrite(os.path.join(output_path, '.'.join([file_name, ext])), image)
tree = ET.ElementTree(xmlRoot)
tree.write('{}/{}.xml'.format(output_path, file_name, ext))
if int(save_box_images):
save_path = '{}/boxes_images/boxed_{}'.format(
output_path, ''.join([file_name, '.', ext]))
draw_box(newBoxes, image, save_path)
async def choose_format(msg: Message, state):
await msg.answer(text='choose new image format', reply_markup=kbs.img_formats())
name = utils.get_file_name(msg=msg)
io = await utils.download(msg=msg)
await states.UserStates.convert.set()
await state.set_data({'name': name, 'io': io})
await db.new_user(msg.from_user.id)
def main_df_split():
""" Main function to split data files """
clear()
pretty_print("You can split the file in equal parts here:", "#")
try:
name = get_file()
pretty_print("How many chunks do you need?", "-")
number = get_int_input()
data_frame = pd.read_excel(name)
split_df = np.array_split(data_frame, number)
for index, dataframe in enumerate(split_df, 1):
file_name = get_file_name()
dataframe.to_excel(f"{file_name}.xlsx", index=False)
pretty_print(f"File {index} {file_name}.xlsx Saved", "*")
pretty_print("Have a Nice Day! - @MrunalSalvi", "&")
sleep(5)
except FileNotFoundError:
clear()
pretty_print("The File Does not Exist.", ":")
pretty_print("Make Sure your place the file in the working directory.",
":")
sleep(2)
main_df_split()
except Exception as log_error:
print("Oops something went wrong...")
print(log_error)
sleep(10)
def main():
parser = argparse.ArgumentParser(prog="mipsal", description='Assemble and link a MIPS assembly program.')
parser.add_argument("files", action="store", nargs="+", type=str, help="list of assembly files to process")
parser.add_argument("--int", action="store_true", default=False, help="output intermediate files")
parser.add_argument("--obj", action="store_true", default=False, help="output object files")
parser.add_argument("-o", action="store", dest="out_name", type=str, default="mips.out", help="override output file name", metavar="file_name")
parser.add_argument("-l", "--link", action="append", help="add file to program when linking. This option can be used more than once", metavar="file_name")
args = parser.parse_args()
obj_code = []
for input_file in args.files:
ints, objs = assembler.assemble(input_file)
obj_code.append(objs)
file_name = utils.get_file_name(input_file)
if args.int:
int_file = file_name + ".int"
utils.write_file_from_list(int_file, ints)
if args.obj:
obj_file = file_name + ".o"
utils.write_file_from_list(obj_file, objs)
if args.link != None:
for link_file in args.link:
obj_code.append([x.strip() for x in utils.read_file_to_list(link_file)])
output = linker.link(obj_code)
utils.write_file_from_list(args.out_name, output)
def train_rnn_model(x_train, y_train, x_val, y_val, params):
rnn_model = model.RNNModel(
max_seq_length=x_train.shape[1],
input_size=params["input_size"],
output_size=params["output_size"],
embed_dim=params["embed_dim"],
emb_trainable=params["emb_trainable"],
model_name=params["model_name"],
hidden_activation=params["hidden_activation"],
out_activation=params["out_activation"],
hidden_dim=params["hidden_dims"][0],
kernel_initializer=params["kernel_initializer"],
kernel_regularizer=params["kernel_regularizer"],
recurrent_regularizer=params["recurrent_regularizer"],
input_dropout=params["input_dropout"],
recurrent_dropout=params["recurrent_dropout"],
rnn_unit_type=params["rnn_unit_type"],
bidirectional=params["bidirectional"],
attention=params["attention"],
embs_matrix=params["embs_matrix"])
history = train_model(rnn_model,
x_train,
y_train,
out_dir=params["out_dir"],
validation_data=(x_val, y_val),
save_checkpoint=params["save_checkpoint"],
n_epochs=params["n_epochs"],
batch_size=params["batch_size"],
verbose=params["verbose"],
early_stopping=params["early_stopping"],
learning_rate=params["learning_rate"],
loss=params["loss"],
ckpt_name_prefix=utils.get_file_name(params))
return utils.extract_results_from_history(history)
def __init__(self, file_path, root_dir, preprocess_bucket=None):
self._rawfile = file_path
self._preprocess_bucket = preprocess_bucket
original_file_name = utils.get_file_name(file_path)
self._preprocess_dir = '/'.join([root_dir, original_file_name])
def export(app, file_path, params, xsi_toolkit):
imp.reload(prim_xform)
imp.reload(prim_mesh)
imp.reload(prim_camera)
imp.reload(prim_light)
imp.reload(prim_hair)
imp.reload(prim_model)
imp.reload(prim_pointcloud)
imp.reload(materials)
imp.reload(utils)
progress_bar = xsi_toolkit.ProgressBar
progress_bar.CancelEnabled = False
progress_bar.Visible = True
stage = Usd.Stage.CreateNew(file_path)
path_head, path_tail = os.path.split(file_path)
path_for_objects = path_head + "\\" + utils.get_file_name(path_tail) + "_objects\\"
utils.add_stage_metadata(stage, params)
opts = params.get("options", None)
if opts is not None:
opts["extension"] = utils.get_file_extension(file_path)
is_materials = True
mats = params.get("materials", None)
if mats is not None:
is_materials = mats.get("is_materials", True)
root_path = ""
materials_opt = {} # some parameters of the exported materials
material_asset_path = None
if is_materials:
materials_path = path_head + "\\" + utils.get_file_name(path_tail) + "_materials." + opts["extension"]
material_asset_path = materials.export_materials(app, params, stage, materials_path, progress_bar)
materials_opt["asset_path"] = material_asset_path
exported_objects = [] # store here ObjectID of exported objects
if len(params["objects_list"]) == 1 and params["objects_list"][0].ObjectID == app.ActiveProject2.ActiveScene.Root.ObjectID:
for obj in app.ActiveProject2.ActiveScene.Root.Children:
export_step(app, params, path_for_objects, stage, obj, exported_objects, materials_opt, root_path, progress_bar)
else:
for obj in params["objects_list"]:
export_step(app, params, path_for_objects, stage, obj, exported_objects, materials_opt, root_path, progress_bar)
stage.Save()
progress_bar.Visible = False
def _upload_to(self, filename):
basedir = os.path.join(settings.MEDIA_ROOT,
self.__class__._meta.app_label[len('everes_'):])
d = self.published_from
if not d:
d = datetime.now()
basedir = os.path.join(basedir, d.strftime('%Y/%m/%d'))
return get_file_name(os.path.join(basedir,
filename))[len(settings.MEDIA_ROOT):]
def save_quit(self):
""" Function to save and quit """
file_name = get_file_name()
self.excel.save(file_name)
for log in self.excel.stack:
pretty_print(log, "/")
pretty_print(f"File Saved as {file_name}_(date).xlsx", ":")
raise SystemExit
def handle_photo(self, bot, update):
folder = 'photo'
chat_id = update.message.chat_id
cascad_file = 'haarcascade_frontalface_default.xml'
file_info = bot.get_file(update.message.photo[-1].file_id)
file_info.download('{}.jpg'.format(chat_id))
face_cascad = cv2.CascadeClassifier(cascad_file)
gray = Image.open('{}.jpg'.format(chat_id)).convert('L')
image = np.array(gray, 'uint8')
faces = face_cascad.detectMultiScale(image,
scaleFactor=1.3,
minNeighbors=7,
minSize=(50, 50))
if len(faces) > 0:
message_count = self.db.get_message_count('image_path', chat_id)
file_name = utils.get_file_name(folder, chat_id, message_count)
copyfile('{}.jpg'.format(chat_id), file_name)
img = Image.open('{}.jpg'.format(chat_id))
for (x, y, w, h) in faces:
# img = Image.open('{}.jpg'.format(chat_id))
draw = ImageDraw.Draw(img)
draw.rectangle((x, y, x + w, y + h), outline='green', width=5)
img.save('{}.jpg'.format(chat_id))
with (open('{}.jpg'.format(chat_id), 'rb')) as file:
bot.send_photo(chat_id, file)
image_name = file_name.split('/')[-1].split('.')[0]
i_count, image_names = self.db.set_path_value(
'image_path', image_name, chat_id)
bot.send_message(chat_id=chat_id,
text='На изображении найдено лицо {} шт\n'
'Изображение сохранено\n'
'У Вас {} фото сообщений с именами\n{}'.format(
len(faces), i_count,
image_names.replace(',', '\n')))
else:
message_info = ''
image_names = self.db.get_image_names(chat_id)
if image_names:
count_image = len(image_names.split(','))
message_info = 'У Вас {} фото сообщений с именами\n{}'.format(
count_image, image_names.replace(',', '\n'))
bot.send_message(chat_id=chat_id,
text='На изображении лицо не найдено \n{}'.format(
message_info))
os.remove('{}.jpg'.format(chat_id))
return 'HANDLE_ECHO'
def create_sample_file(p_source_dir, p_destination_dir, p_source_file,
p_file_type):
utils.log(logger, "Begin - create_sample_file -")
samplecountlines = 0
source_file = utils.get_file_name(str(p_source_file))
with open(p_destination_dir + "SAM_" + source_file + p_file_type,
'wb') as xfile:
utils.log(logger, "Reading file SAMPLE: " + p_source_file)
csvwriter = csv.writer(xfile,
delimiter='\t',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
with open(p_source_dir + p_source_file, 'rb') as samplefile:
INDfilelist = []
for line in samplefile:
samplecountlines = samplecountlines + 1
if samplecountlines <= 2:
seq = str(samplecountlines * (-1)).split()
columns = line.split()
csvwriter.writerow(seq + columns)
#Start counting individuals
if samplecountlines > 2:
seq = str(samplecountlines - 2).split()
columns = line.split()
col01 = columns[0:2] #to create the file ID
csvwriter.writerow(seq + columns)
#Create empty INDIVIDUAL file
INDfilename = create_individuals_file(
p_destination_dir,
seq[0] + "_" + col01[0] + "_" + col01[1], p_file_type)
#Create list with Individuals file
INDfilelist.append(INDfilename)
samplefile.close()
xfile.close()
utils.log(logger, "SAMPLE file lines: " + str(samplecountlines))
utils.log(logger, "End - create_sample_file -")
return INDfilelist
def concat_data_main():
""" Main Concat Data Function """
clear()
pretty_print("You can combine two or more excel files here:", "#")
pretty_print("How many files do you want to combine:", "-")
file_list = []
try:
number = get_int_input()
if number <= 1:
raise ValueError
if number >= 10:
raise ArithmeticError
for i in range(number):
name = get_file()
data_frame = pd.read_excel(name)
file_list.append(data_frame)
clear()
full_df = concat_ordered_columns(file_list)
new_name = get_file_name()
full_df.to_excel(f"{new_name}.xlsx", index=False)
pretty_print(f"File Saved as {new_name}.xlsx", "-")
pretty_print("Have a Nice Day! - @Mrunal", "&")
sleep(5)
except FileNotFoundError:
clear()
pretty_print("The File Does not Exist.", ":")
pretty_print("Make Sure your place the file in the working directory.", ":")
sleep(2)
concat_data_main()
except ValueError:
pretty_print("You can't combine a single file or no file... How sad!", ":")
pretty_print("Try Again!", ":")
sleep(5)
concat_data_main()
except ArithmeticError:
pretty_print("I can combine more than 10 excel files together but I just don't want to ;-)", ":")
pretty_print("Try Again", ":")
sleep(5)
concat_data_main()
except Exception as log_error:
print("Oops something went wrong")
print(log_error)
sleep(10)
def __call__(self):
self.check_all_paths()
if self.args.off:
image_list = utils.get_image_set(self.row_image_path)
for image in tqdm(image_list):
shutil.copy(
image,
os.path.join(self.training_image_path,
utils.get_file_name(image)))
shutil.copy(
self.row_label[0],
os.path.join(self.configer['traininglabelPath'],
utils.get_file_name(self.row_label[0])))
if self.args.testSamples:
self.split_dev_dataset(self.args.testSamples)
if self.args.massCrop:
self.save_bordercroped_images(self.args.borderCropRate)
self.save_centercropsed_images()
self.pad_images(self.args.padBorderSize)
if self.args.dataBalance:
self.extend_dataset(self.args.dataBalance)
def download_css_files(self):
text = open(self.file_path, 'rb+').read().decode('utf-8')
soup = BeautifulSoup(text, "lxml")
# find css files
css_list = soup.find_all(attrs={"type": "text/css"})
for css in css_list:
if 'svgtextcss' in css.get('href'):
url = 'http:' + css.get('href')
request.request(url, './css/', 'css')
self.parse_css_file('./css/' + utils.get_file_name(url))
print("Css files have been downloaded and parsed.")
def convert_cols_to_lines(p_source_dir, p_source_file, p_destination_dir,
p_dest_file_list, p_individualsposlist,
p_gen_column):
utils.log(logger, "Begin - convert_gen_cols_to_ind_lines - ")
positionindex = p_individualsposlist.index(p_gen_column)
regex = r"^{0}.*{1}$".format(
p_destination_dir + "IND_" + str(positionindex + 1) + "_",
destination_file_type)
p_indfilename = utils.find_file_in_list(p_dest_file_list, regex)
source_file = utils.get_file_name(str(p_source_file))
try:
col = int(p_gen_column)
except:
e = sys.exc_info()[0]
utils.log(logger, e)
#Open individuals file
with open(p_indfilename, 'a') as indfile:
utils.log(logger, "Writing IND .tsv file: " + p_indfilename)
csvwriter = csv.writer(indfile,
delimiter='\t',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
sequence = 0
with gzip.open(p_source_dir + p_source_file, 'rb') as genfile:
for line in genfile: #reads line by line .gen file.
#readlines() loads full .gen file into memory and split in lines. To many threads
# or very big files can cause memory overflow.
#for line in genfile.readlines():
sequence = sequence + 1
seq = str(sequence).split()
columns = line.split()
csvwriter.writerow([source_file] + seq + columns[col:col + 3])
indfile.close()
utils.log(logger, "Lines in source file: " + str(sequence))
genfile.close()
utils.log(logger, "End - convert_gen_cols_to_ind_lines - ")
return
def create_individuals_sample_files(p_source_dir, p_destination_dir,
p_source_file, p_file_type):
utils.log(logger, "Begin - create_individuals_sample_files -")
samplecountlines = 0
source_file = utils.get_file_name(str(p_source_file))
INDfilelist = []
with open(p_source_dir + p_source_file, 'rb') as samplefile:
for line in samplefile:
samplecountlines = samplecountlines + 1
columns = line.split()
if samplecountlines == 1:
headerline = columns[:]
elif samplecountlines == 2:
datatypeline = columns[:]
else:
individualline = samplecountlines - 2
with open(
p_destination_dir + "SAM_" + str(individualline) +
"_" + str(columns[0]) + "_" + str(columns[1]) +
p_file_type, 'wb') as xfile:
csvwriter = csv.writer(xfile,
delimiter='\t',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
for i in range(0, len(columns)):
csvwriter.writerow([headerline[i]] +
[datatypeline[i]] + [columns[i]])
#Create empty INDIVIDUAL file
INDfilename = create_individuals_file(
p_destination_dir,
str(individualline) + "_" + columns[0] + "_" +
columns[1], p_file_type)
#Create list with Individuals file
INDfilelist.append(INDfilename)
xfile.close()
samplefile.close()
utils.log(logger, "SAMPLE file lines: " + str(samplecountlines))
utils.log(logger, "End - create_individuals_sample_files -")
return INDfilelist
def process_image(file_path, output_path, x, y, save_box_images, mode):
(base_dir, file_name, ext) = get_file_name(file_path)
image_path = '{}/{}.{}'.format(base_dir, file_name, ext)
xml = '{}/{}.xml'.format(base_dir, file_name)
try:
resize(image_path,
xml, (x, y),
output_path,
mode,
save_box_images=save_box_images)
except Exception as e:
print('[ERROR] error with {}\n file: {}'.format(image_path, e))
print('--------------------------------------------------')
def split_hdmi(hdmi_path: str,
json_timing: dict,
output_dir: str,
post_fix: str = "",
audio: bool = True,
crop_coords_per_slide: list = []):
"""
Takes an hdmi video as input and splits it into multiple videos.
The post_fix string is used to append to the end of the output files.
Audio defaults to true: April 2019 decision to always include audio in slides (to be muted by frontend, sometimes, depending on layout).
"""
# Generate the output directory if it does not exist.
make_directory(output_dir)
OFFSETDELAY = 0.4
# check arg
if len(crop_coords_per_slide) > 0:
assert len(crop_coords_per_slide) == len(json_timing['allId']), str(
len(crop_coords_per_slide)) + ' vs ' + str(
len(json_timing['allId']))
# Start splitting the videos based off the timings.
# NOTE: The videos will be in the order dictated by allId!!!!
finalidx = int(len(json_timing['allId'])) - 1
for index, slide_id in enumerate(json_timing['allId']):
# Get timing attributes
time_start = float(
json_timing['byId'][slide_id]['start']) + OFFSETDELAY
time_end = float(json_timing['byId'][slide_id]['end']) + OFFSETDELAY
duration = time_end - time_start
# Generate the output file name
output_file = append_slash(output_dir) + append_postfix(
get_file_name(hdmi_path), post_fix + str(index))
# Start splitting
print('Splitting HDMI video to file %s' % output_file)
docrop = None
if len(crop_coords_per_slide) > 0:
docrop = crop_coords_per_slide[index]
assert isinstance(docrop,
dict), str(index) + ': ' + str(type(docrop))
ffmpeg_split(hdmi_path,
time_start,
duration,
output_file,
audio,
is_final_end_of_video=(index >= finalidx),
docrop=docrop)
def main():
parser = argparse.ArgumentParser(prog="mipsa", description='Assemble a MIPS assembly program. Outputs an object file for every input file.')
parser.add_argument("files", action="store", nargs="+", type=str, help="list of assembly files to process")
parser.add_argument("--int", action="store_true", default=False, help="output intermediate files")
args = parser.parse_args()
for input_file in args.files:
ints, objs = assemble(input_file)
file_name = utils.get_file_name(input_file)
if args.int:
int_file = file_name + ".int"
utils.write_file_from_list(int_file, ints)
obj_file = file_name + ".o"
utils.write_file_from_list(obj_file, objs)
def main():
parser = argparse.ArgumentParser(
prog="mipsal",
description='Assemble and link a MIPS assembly program.')
parser.add_argument("files",
action="store",
nargs="+",
type=str,
help="list of assembly files to process")
parser.add_argument("--int",
action="store_true",
default=False,
help="output intermediate files")
parser.add_argument("--obj",
action="store_true",
default=False,
help="output object files")
parser.add_argument("-o",
action="store",
dest="out_name",
type=str,
default="mips.out",
help="override output file name",
metavar="file_name")
parser.add_argument(
"-l",
"--link",
action="append",
help=
"add file to program when linking. This option can be used more than once",
metavar="file_name")
args = parser.parse_args()
obj_code = []
for input_file in args.files:
ints, objs = assembler.assemble(input_file)
obj_code.append(objs)
file_name = utils.get_file_name(input_file)
if args.int:
int_file = file_name + ".int"
utils.write_file_from_list(int_file, ints)
if args.obj:
obj_file = file_name + ".o"
utils.write_file_from_list(obj_file, objs)
if args.link != None:
for link_file in args.link:
obj_code.append(
[x.strip() for x in utils.read_file_to_list(link_file)])
output = linker.link(obj_code)
utils.write_file_from_list(args.out_name, output)
def staging(infile):
filename = get_file_name(infile)
path = get_path_to_file(infile)
response = None
try:
response = urllib.request.urlopen(infile)
except e:
print(e.reason)
content = response.read()
with open("{}".format(filename), "wb") as data:
data.write(content)
return filename
def convert_cols_to_lines(p_source_dir,p_source_file,p_destination_dir,p_dest_file_list, p_individualsposlist, p_gen_column):
utils.log(logger,"Begin - convert_gen_cols_to_ind_lines - ")
positionindex = p_individualsposlist.index(p_gen_column)
regex = r"^{0}.*{1}$".format(p_destination_dir+"IND_"+str(positionindex+1)+"_",destination_file_type)
p_indfilename = utils.find_file_in_list(p_dest_file_list,regex)
source_file = utils.get_file_name(str(p_source_file))
try:
col = int(p_gen_column)
except:
e = sys.exc_info()[0]
utils.log(logger,e)
#Open individuals file
with open(p_indfilename,'a') as indfile:
utils.log(logger,"Writing IND .tsv file: "+ p_indfilename)
csvwriter = csv.writer(indfile,delimiter='\t',quotechar='"', quoting=csv.QUOTE_MINIMAL)
sequence = 0
with gzip.open(p_source_dir+p_source_file,'rb') as genfile:
for line in genfile: #reads line by line .gen file.
#readlines() loads full .gen file into memory and split in lines. To many threads
# or very big files can cause memory overflow.
#for line in genfile.readlines():
sequence=sequence+1
seq=str(sequence).split()
columns=line.split()
csvwriter.writerow([source_file]+seq+columns[col:col+3])
indfile.close()
utils.log(logger,"Lines in source file: "+ str(sequence))
genfile.close()
utils.log(logger,"End - convert_gen_cols_to_ind_lines - ")
return
def create_sample_file(p_source_dir,p_destination_dir, p_source_file, p_file_type):
utils.log(logger,"Begin - create_sample_file -")
samplecountlines = 0
source_file = utils.get_file_name(str(p_source_file))
with open(p_destination_dir+"SAM_"+source_file+p_file_type, 'wb') as xfile:
utils.log(logger,"Reading file SAMPLE: " + p_source_file)
csvwriter = csv.writer(xfile,delimiter='\t',quotechar='"', quoting=csv.QUOTE_MINIMAL)
with open(p_source_dir+p_source_file,'rb') as samplefile:
INDfilelist = []
for line in samplefile:
samplecountlines=samplecountlines+1
if samplecountlines <= 2:
seq=str(samplecountlines * (-1)).split()
columns=line.split()
csvwriter.writerow(seq+columns)
#Start counting individuals
if samplecountlines > 2:
seq=str(samplecountlines-2).split()
columns=line.split()
col01= columns[0:2] #to create the file ID
csvwriter.writerow(seq+columns)
#Create empty INDIVIDUAL file
INDfilename = create_individuals_file(p_destination_dir, seq[0]+"_"+col01[0]+"_"+col01[1], p_file_type)
#Create list with Individuals file
INDfilelist.append(INDfilename)
samplefile.close()
xfile.close()
utils.log(logger,"SAMPLE file lines: "+ str(samplecountlines))
utils.log(logger,"End - create_sample_file -")
return INDfilelist
def create_individuals_sample_files(p_source_dir,p_destination_dir, p_source_file, p_file_type):
utils.log(logger,"Begin - create_individuals_sample_files -")
samplecountlines = 0
source_file = utils.get_file_name(str(p_source_file))
INDfilelist = []
with open(p_source_dir+p_source_file,'rb') as samplefile:
for line in samplefile:
samplecountlines = samplecountlines + 1
columns = line.split()
if samplecountlines == 1:
headerline = columns[:]
elif samplecountlines == 2:
datatypeline = columns[:]
else:
individualline = samplecountlines - 2
with open(p_destination_dir+"SAM_"+str(individualline)+"_"+str(columns[0])+"_"+str(columns[1])+p_file_type, 'wb') as xfile:
csvwriter = csv.writer(xfile,delimiter='\t',quotechar='"', quoting=csv.QUOTE_MINIMAL)
for i in range(0, len(columns)):
csvwriter.writerow([headerline[i]]+[datatypeline[i]]+[columns[i]])
#Create empty INDIVIDUAL file
INDfilename = create_individuals_file(p_destination_dir, str(individualline)+"_"+columns[0]+"_"+columns[1], p_file_type)
#Create list with Individuals file
INDfilelist.append(INDfilename)
xfile.close()
samplefile.close()
utils.log(logger,"SAMPLE file lines: "+ str(samplecountlines))
utils.log(logger,"End - create_individuals_sample_files -")
return INDfilelist
def create_snp_file(p_source_dir,p_destination_dir, p_source_file_type, p_dest_file_type):
utils.log(logger,"Begin - Create SNP file -")
filename = p_destination_dir+"SNP"+p_dest_file_type
open(filename, 'w').close()
for file_list in sorted(os.listdir(p_source_dir)):
if fnmatch.fnmatch(file_list,'*'+p_source_file_type):
with gzip.open(p_source_dir+file_list,'rb') as genfile:
sequence=0
gencountlines=0
utils.log(logger,"Reading file GEN: " + file_list)
with open(filename,'ab') as SNPfile:
csvwriter = csv.writer(SNPfile,delimiter='\t',quotechar='"', quoting=csv.QUOTE_MINIMAL)
#readlines() Loads full .gen file into memory and split in lines. To many threads
# or very big files can cause memory overflow.
#for line in genfile.readlines():
for line in genfile: #Read file line by line
gencountlines=gencountlines+1
columns=line.split()
col05=columns[0:5]
source_file = utils.get_file_name(file_list)
sequence=sequence+1
seq=str(sequence).split()
csvwriter.writerow([source_file]+seq+col05)
SNPfile.close()
genfile.close()
utils.log(logger,"End - Create SNP file -")
return
def from_file(cls, file_path):
preset = cls()
preset.name = utils.get_file_name(file_path)
with open(file_path, 'rb') as ifile:
stream = BinaryStream(ifile)
preset.header['strings'] = []
# Header portion -----------------------------------
preset.header['version'] = stream.readInt32()
stream.readBytes(16)
strings = preset.header['strings']
strings.append(stream.readNullTerminatedString())
stream.readBytes(4)
# Guessing that this `04 00 00 00` refers to how many strings are
# to follow. If not, then these bytes are unknown and there are
# just 4 strings following regardless
for i in xrange(stream.readInt32()):
strings.append(stream.readNullTerminatedString())
# End of Header portion ----------------------------
# Unknown Bytes ------------------------------------
stream.readBytes(12)
unknown_bytes = stream.readInt64()
# End of Unknown Bytes -----------------------------
# Start of Structured Data -------------------------
stream.read_pair()
struct_property_size = stream.readInt64()
stream.readNullTerminatedString()
while True:
data = read_variable_data(stream)
if data[0] is not None:
preset.character_setting[data[0]] = data[1]
else:
break
# End of Structured Data ---------------------------
return preset
def load_sdrs(start_idx, end_idx, exp_name):
# Params
input_width = 2048 * 32
active_cells_weight = 0
predicted_active_cells_weight = 1
network_config = 'sp=True_tm=True_tp=False_SDRClassifier'
# load traces
file_name = get_file_name(exp_name, network_config)
traces = loadTraces(file_name)
num_records = len(traces['sensorValue'])
# start and end
if start_idx < 0:
start = num_records + start_idx
else:
start = start_idx
if end_idx < 0:
end = num_records + end_idx
else:
end = end_idx
# input data
sensor_values = traces['sensorValue'][start:end]
categories = traces['actualCategory'][start:end]
active_cells = traces['tmActiveCells'][start:end]
predicted_active_cells = traces['tmPredictedActiveCells'][start:end]
# generate sdrs to cluster
active_cells_sdrs = convert_to_sdrs(active_cells, input_width)
predicted_active_cells_sdrs = np.array(
convert_to_sdrs(predicted_active_cells, input_width))
sdrs = (float(active_cells_weight) * np.array(active_cells_sdrs) +
float(predicted_active_cells_weight) * predicted_active_cells_sdrs)
return sdrs, categories
def __init__(self, inputf, outputf):
self.input = inputf
self.output = os.path.join(outputf, "{}.pdf".format(get_file_name(inputf)))
self.datas, self.options = parse_xml_for_pdf(inputf)
def main():
distance_functions = [euclidian_distance]
clustering_classes = [PerfectClustering, OnlineClusteringV2]
network_config = "sp=True_tm=True_tp=False_SDRClassifier"
exp_names = [
"body_acc_x",
"binary_ampl=10.0_mean=0.0_noise=0.0",
"binary_ampl=10.0_mean=0.0_noise=1.0",
"sensortag_z",
]
# Exp params
moving_average_window = 2 # for all moving averages of the experiment
ClusteringClass = clustering_classes[1]
distance_func = distance_functions[0]
exp_name = exp_names[0]
start_idx = 1000
end_idx = 12000
input_width = 2048 * 32
active_cells_weight = 0
predicted_active_cells_weight = 10
max_num_clusters = 3
num_cluster_snapshots = 1
show_plots = True
distance_matrix_ignore_noise = True # whether to ignore label 0 (noise)
# Clean an create output directory for the graphs
plots_output_dir = "plots/%s" % exp_name
if os.path.exists(plots_output_dir):
shutil.rmtree(plots_output_dir)
os.makedirs(plots_output_dir)
# load traces
file_name = get_file_name(exp_name, network_config)
traces = loadTraces(file_name)
num_records = len(traces["sensorValue"])
# start and end for the x axis of the graphs
if start_idx < 0:
start = num_records + start_idx
else:
start = start_idx
if end_idx < 0:
end = num_records + end_idx
else:
end = end_idx
xlim = [0, end - start]
# input data
sensor_values = traces["sensorValue"][start:end]
categories = traces["actualCategory"][start:end]
active_cells = traces["tmActiveCells"][start:end]
predicted_active_cells = traces["tmPredictedActiveCells"][start:end]
raw_anomaly_scores = traces["rawAnomalyScore"][start:end]
anomaly_scores = []
anomaly_score_ma = 0.0
for raw_anomaly_score in raw_anomaly_scores:
anomaly_score_ma = moving_average(anomaly_score_ma, raw_anomaly_score, moving_average_window)
anomaly_scores.append(anomaly_score_ma)
# generate sdrs to cluster
active_cells_sdrs = convert_to_sdrs(active_cells, input_width)
predicted_active_cells_sdrs = np.array(convert_to_sdrs(predicted_active_cells, input_width))
sdrs = (
float(active_cells_weight) * np.array(active_cells_sdrs)
+ float(predicted_active_cells_weight) * predicted_active_cells_sdrs
)
# list of timesteps specifying when a snapshot of the clusters will be taken
step = (end - start) / num_cluster_snapshots - 1
cluster_snapshot_indices = range(step, end - start, step)
# run clustering
(clustering_accuracies, cluster_snapshots, closest_cluster_history) = run(
sdrs,
categories,
anomaly_scores,
distance_func,
moving_average_window,
max_num_clusters,
ClusteringClass,
cluster_snapshot_indices,
)
# cluster_categories = []
# for c in closest_cluster_history:
# if c is not None:
# cluster_categories.append(c.label_distribution()[0]['label'])
# plot cluster assignments over time
for i in range(num_cluster_snapshots):
clusters = cluster_snapshots[i]
snapshot_index = cluster_snapshot_indices[i]
plot_cluster_assignments(plots_output_dir, clusters, snapshot_index)
# plot inter-cluster distance matrix
# plot_id = 'inter-cluster_t=%s' % snapshot_index
# plot_inter_sequence_distances(plots_output_dir,
# plot_id,
# distance_func,
# sdrs[:snapshot_index],
# cluster_categories[:snapshot_index],
# distance_matrix_ignore_noise)
# plot inter-category distance matrix
plot_id = "inter-category_t=%s " % snapshot_index
plot_inter_sequence_distances(
plots_output_dir,
plot_id,
distance_func,
sdrs[:snapshot_index],
categories[:snapshot_index],
distance_matrix_ignore_noise,
)
# plot clustering accuracy over time
plot_id = "file=%s | moving_average_window=%s" % (exp_name, moving_average_window)
plot_accuracy(plots_output_dir, plot_id, sensor_values, categories, anomaly_scores, clustering_accuracies, xlim)
if show_plots:
plt.show()
def _on_reduce_ack(self, server, type, nick, data):
"""
The method is triggered whenever a master wants to communicate that a
reduce job was succesfully computed. The payload structure is a 2-tuple:
- The first element is an integer telling the reduce_idx
- The second element is a list in where:
- The first element is the output file (fid, fsize)
- The other elements are the inputs file that can be safely deleted
(fid, fsize)
"""
if not nick in server.masters:
self.__send_data('registration-needed')
return
reduce_idx = data[0]
to_add = data[1][0] # NB: This is a tuple (fid, fsize)
to_delete = data[1][1:] # NB: This is instead a sequence of [fid, ..]
if nick in server.reduce_mark:
server.reduce_mark.remove(nick)
jobs = server.reduce_dict[nick][reduce_idx]
server.info("Received %s from %s" % (str(data), nick))
server.info("Jobs for the reducer %d is %s" % (reduce_idx, str(jobs)))
opos = 0
dpos = 0
while opos < len(jobs):
ofid, ofsize = jobs[opos]
dpos = 0
while dpos < len(to_delete):
dfid = to_delete[dpos]
if ofid == dfid:
fname = get_file_name(server.path, reduce_idx, dfid)
if server.use_dfs:
try:
server.work_queue.fs.nukeFile(fname)
server.info("Nuking map file %s from DFS" % fname)
except:
pass
else:
server.info("Removing map output file %s" % fname)
os.unlink(fname)
del to_delete[dpos]
del jobs[opos]
opos -= 1
break
dpos += 1
opos += 1
# Execute it anyway. It will be buffered.
server.retrieve_file(nick, reduce_idx, tuple(to_add))
if server.status.phase == server.status.PHASE_MERGE:
server.reduce_dict[nick] = None
else:
jobs.append(tuple(to_add))
server.status.reduce_assigned += 1
server.status.reduce_completed += 1
server.status.reduce_file += 1
server.status.reduce_file_size += to_add[1]
server.status.add_graph_point()
if len(to_delete) > 0:
server.error("Failed to remove reduce files %s" % str(to_delete))
self.__send_data('reduce-ack-fail', nick, data)
def retrieve_file(self, nick, reduce_idx, file):
fid, fsize = file
fname = get_file_name(self.path, reduce_idx, fid)
self.reduce_files[reduce_idx] = (nick, fname, fsize)
def main():
distance_functions = [euclidian_distance]
clustering_classes = [PerfectClustering, OnlineClustering]
network_config = 'sp=True_tm=True_tp=False_SDRClassifier'
exp_names = ['binary_ampl=10.0_mean=0.0_noise=0.0',
'binary_ampl=10.0_mean=0.0_noise=1.0',
'sensortag_z']
# Exp params
moving_average_window = 1 # for all moving averages of the experiment
ClusteringClass = clustering_classes[0]
distance_func = distance_functions[0]
exp_name = exp_names[0]
start_idx = 0
end_idx = 100
input_width = 2048 * 32
active_cells_weight = 0
predicted_active_cells_weight = 1
max_num_clusters = 3
num_cluster_snapshots = 2
show_plots = False
distance_matrix_ignore_noise = True # whether to ignore label 0 (noise)
# Clean an create output directory for the graphs
plots_output_dir = 'plots/%s' % exp_name
if os.path.exists(plots_output_dir):
shutil.rmtree(plots_output_dir)
os.makedirs(plots_output_dir)
# load traces
file_name = get_file_name(exp_name, network_config)
traces = loadTraces(file_name)
sensor_values = traces['sensorValue'][start_idx:end_idx]
categories = traces['actualCategory'][start_idx:end_idx]
raw_anomaly_scores = traces['rawAnomalyScore'][start_idx:end_idx]
anomaly_scores = []
anomaly_score_ma = 0.0
for raw_anomaly_score in raw_anomaly_scores:
anomaly_score_ma = moving_average(anomaly_score_ma,
raw_anomaly_score,
moving_average_window)
anomaly_scores.append(anomaly_score_ma)
active_cells = traces['tmActiveCells'][start_idx:end_idx]
predicted_active_cells = traces['tmPredictedActiveCells'][start_idx:end_idx]
# generate sdrs to cluster
active_cells_sdrs = convert_to_sdrs(active_cells, input_width)
predicted_activeCells_sdrs = np.array(convert_to_sdrs(predicted_active_cells,
input_width))
sdrs = (active_cells_weight * np.array(active_cells_sdrs) +
predicted_active_cells_weight * predicted_activeCells_sdrs)
# start and end for the x axis of the graphs
start = start_idx
if end_idx < 0:
end = len(sdrs) - end_idx - 1
else:
end = end_idx
xlim = [start, end]
# list of timesteps specifying when a snapshot of the clusters will be taken
step = (end - start) / num_cluster_snapshots - 1
cluster_snapshot_indices = range(start + step, end, step)
# run clustering
(clustering_accuracies,
cluster_snapshots,
closest_cluster_history) = run(sdrs,
categories,
distance_func,
moving_average_window,
max_num_clusters,
ClusteringClass,
cluster_snapshot_indices)
# plot cluster assignments over time
for i in range(num_cluster_snapshots):
clusters = cluster_snapshots[i]
plot_cluster_assignments(plots_output_dir, clusters, cluster_snapshot_indices[i])
# plot inter-cluster distance matrix
cluster_ids = [c.id for c in closest_cluster_history if c is not None]
plot_id = 'inter-cluster_t=%s' % cluster_snapshot_indices[i]
plot_inter_sequence_distances(plots_output_dir,
plot_id,
distance_func,
sdrs[:cluster_snapshot_indices[i]],
cluster_ids[:cluster_snapshot_indices[i]],
distance_matrix_ignore_noise)
# plot inter-category distance matrix
plot_id = 'inter-category_t=%s ' % cluster_snapshot_indices[i]
plot_inter_sequence_distances(plots_output_dir,
plot_id,
distance_func,
sdrs[:cluster_snapshot_indices[i]],
categories[:cluster_snapshot_indices[i]],
distance_matrix_ignore_noise)
# plot clustering accuracy over time
plot_id = 'file=%s | moving_average_window=%s' % (exp_name,
moving_average_window)
plot_accuracy(plots_output_dir,
plot_id,
sensor_values,
categories,
anomaly_scores,
clustering_accuracies,
xlim)
if show_plots:
plt.show()
