def plot_accperstim(title, data_to_analyze, stim_ids='stimulus', stims_all=None, label_count_cutoff=50, extract_stim_names=True): ''' percent correct broken out by stimulus and day. Parameters: ----------- title : str the plot title data_to_analyze : pandas DataFrame filtered data to plot. Can be a slice, a copy is made anyways. stim_ids : str label of the column to group-by. stims_all : None or list-like order of stims. must match values in stim_ids label_count_cutoff : int max number of stimuli labels. If below this value will sort stim_ids by class. extract_stim_names : boolean whether to extract stimuli names from full stimuli paths. If true, ignores stim_ids. ''' data_to_analyze = data_to_analyze.copy() if extract_stim_names: stim_ids = 'stim_name' utils.extract_filename(data_to_analyze, target=stim_ids) data_to_analyze['date'] = data_to_analyze.index.date blocked = data_to_analyze.groupby(['date', stim_ids]) aggregated = pd.DataFrame( blocked.agg({ 'correct': lambda x: np.mean(x.astype(float)) }).to_records()) pivoted = aggregated.pivot(stim_ids, 'date', 'correct') if stims_all: yticklabels = stims_all elif len(pivoted) < label_count_cutoff: yticklabels = data_to_analyze.groupby( ['class_', stim_ids]).index.unique().index.get_level_values(stim_ids).values else: yticklabels = int(len(pivoted) / label_count_cutoff) cmap = sns.diverging_palette(15, 250, as_cmap=True) cmap.set_bad(color='k', alpha=0.5) plt.figure() g = sns.heatmap(pivoted, vmin=0, vmax=1, cmap=cmap, xticklabels=_date_labels(list(pivoted.keys()).values), yticklabels=yticklabels) g.set_title(title)
def test_extract_filename(self): self.assertEqual(utils.extract_filename("/home/user/teste.txt.txt"), "/home/user/teste.txt") self.assertEqual(utils.extract_filename("/home/user/teste"), "/home/user/teste") self.assertEqual(utils.extract_filename("teste"), "teste") self.assertEqual(utils.extract_filename("teste.txt"), "teste") self.assertEqual(utils.extract_filename("teste.txt.txt"), "teste.txt") self.assertRaises(Exception, utils.extract_filename, (".txt")) self.assertRaises(Exception, utils.extract_filename, ("")) self.assertRaises(Exception, utils.extract_filename, ())
def testMutatedData(self, mutation, seed_files): mutator = self.op_dict[mutation].mOP() seed_dep = mutator.__seed_dependency__[0] resource_file = None for i in seed_files: if "." + seed_dep in i: seed_file = i break output = { 'filename': utils.extract_filename(seed_file), 'fileext': utils.extract_fileext(seed_file), 'filetype': utils.extract_filetype(seed_file), 'content': utils.extract_content(seed_file) } origin = { 'filename': utils.extract_filename(seed_file), 'fileext': utils.extract_fileext(seed_file), 'filetype': utils.extract_filetype(seed_file), 'content': utils.extract_content(seed_file) } mutator.operation(output, seed_file, resource_file) seed_type = seed_file.split(".")[-1] write_content(seed_type, output) """ if output['filename'] != origin['filename']: print "[+] Mutation succeed ( {} ) - filename".format(mutation) return True el """ if output['fileext'] != origin['fileext']: print "[+] Mutation succeed ( {} ) - fileext( {} -> {} )".format( mutation, origin["fileext"], output["fileext"]) return True elif output['filetype'] != origin['filetype']: print "[+] Mutation succeed ( {} ) - filetype( {} -> {} )".format( mutation, origin["filetype"], output["filetype"]) return True elif output['content'] != origin['content']: print "[+] Mutation succeed ( {} ) - content( {} b -> {} b )".format( mutation, len(origin["content"]), len(output["content"])) elif output['filename'][0] == '.': print "[+] Mutation succeed ( {} ) - filename( {} -> {} )".format( mutation, origin["filename"], output["filename"]) return True return False
def operation(self, output, seed_file, resource_file=None): output['fileext'] = '' if output['filename'] != None and len(output['filename']) > 0: filename = output['filename'] else: filename = utils.extract_filename(seed_file) output['filename'] = filename + '_M7'
def _downloadWikiAttachment(self, entity, wiki, filename, dest_dir=None): """Download a file attached to a wiki page (experimental)""" # TODO can wiki pages exist without being associated with an Entity? ## build URL wiki_id = wiki['id'] if 'id' in wiki else str(wiki) entity_id = entity['id'] if 'id' in entity else str(entity) url = "%s/entity/%s/wiki/%s/attachment?fileName=%s" % (self.repoEndpoint, entity_id, wiki_id, filename,) ## we expect to be redirected to a signed S3 URL ## TODO how will external URLs be handled? response = requests.get(url, headers=self.headers, allow_redirects=False) if response.status_code in [301,302,303,307,308]: url = response.headers['location'] # print url headers = {'sessionToken':self.sessionToken} response = requests.get(url, headers=headers, stream=True) response.raise_for_status() ## stream file to disk filename = utils.extract_filename(response.headers['content-disposition']) if dest_dir: filename = os.path.join(dest_dir, filename) with open(filename, "wb") as f: data = response.raw.read(1024) while data: f.write(data) data = response.raw.read(1024) return os.path.abspath(filename)
def operation(self, output, seed_file, resource_file=None): output['content'] = output['content'].replace('<?php', '<?') if output['filename'] != None and len(output['filename']) > 0: filename = output['filename'] else: filename = utils.extract_filename(seed_file) output['filename'] = filename + '_M5'
def dependency_paths(self, filepath: str) -> set: # Ensure that filepath is valid dependencies = set() if filepath not in self.graph: return dependencies # Initialize root node as filepath name = utils.extract_filename(filepath) root = Node(name=name, ID=filepath) context = [root] visited = set() ## DFS through graph and add path to dependencies list each time we encounter a leaf def visit(u: TreeSitterNode): if self.graph[u.ID] and u.ID not in visited: visited.add(u.ID) for v in self.graph[u.ID]: context.append(v) visit(v) context.pop() else: dependencies.add(tuple(context)) visit(root) return dependencies
def segment_transform(filelist, conf, is_train=True): not_change_pos_keys = {"", "n", "l", "nl"} img_resize = conf["img_resize"] for filepath in filelist: file_split = extract_filename(filepath) filename = file_split["filename"] imgname = file_split["imgname"] transform_type = imgname.split("_")[-1] if is_train: org_dir = conf["train_data_dir"] prep_dir = conf["prep_train_dir"] else: org_dir = conf["test_data_dir"] prep_dir = conf["prep_test_dir"] if transform_type in not_change_pos_keys: org_xml_file = "_".join(imgname.split("_")[:-2]) + ".xml" new_xml_file = imgname + ".xml" org_xml_path = os.path.join(org_dir, org_xml_file) mytree, myroot = load_base_xml(org_xml_path) # iterating throught the price values. for p11 in myroot.iter('filename'): # updates the price value p11.text = filename # iterating throught the price values. for p12 in myroot.iter('path'): # updates the price value p12.text = os.path.join(prep_dir, org_xml_file) # iterating throught the price values. for p11 in myroot.iter('width'): # updates the price value ratio_changed = int(p11.text) / img_resize p11.text = str(img_resize) # iterating throught the price values. for p12 in myroot.iter('height'): # updates the price value p12.text = str(img_resize) # iterating throught the price values. for p1 in myroot.iter('xmin'): # updates the price value p1.text = str(int(p1.text) / ratio_changed) for p2 in myroot.iter('xmax'): # updates the price value p2.text = str(int(p2.text) / ratio_changed) for p3 in myroot.iter('ymin'): # updates the price value p3.text = str(int(p3.text) / ratio_changed) for p4 in myroot.iter('ymax'): # updates the price value p4.text = str(int(p4.text) / ratio_changed) else: # todo position img change raise () new_xml_path = os.path.join(prep_dir, new_xml_file) write_base_cml(new_xml_path, mytree)
def operation(self, output, seed_file, resource_file=None): if output['filename'] != None and len(output['filename']) > 0: filename = output['filename'] else: filename = utils.extract_filename(seed_file) #change = lambda a : chr(ord(a)+1) output['filename'] = filename + '_M10' #output['fileext'] = ''.join(map(change,list(output['fileext']))) output['fileext'] = 'fuse'
def augment_images(file_list, conf, mode): """Generate augment images""" for file_path, n in file_list: img = read_img(file_path, conf["grayscale"]) height, width = img.shape[:2] if (height, width) != (conf["img_resize"], conf["img_resize"]): if height != width and conf["fill_square"]: img = fill_squre(img, conf["fill_square_color"]) img = cv2.resize(img, (conf["img_resize"], conf["img_resize"])) # IMAGE AUGMENTATION file_split = extract_filename(file_path) img_name = file_split["imgname"] ext = file_split["ext"] if mode == "train" or mode == "val": if mode == "train": prep_dir = conf["prep_train_dir"] else: prep_dir = conf["prep_val_dir"] for i in range(n): img_varied = img.copy() varied_imgname = '{}_{:0>3d}_'.format(img_name, i) if random.random() < conf["p_noise"]: # not change pos fn img_varied = random_noise(img_varied, conf["noise_vari"]) varied_imgname += 'n' if random.random() < conf["p_light"]: # not change pos fn img_varied = random_light(img_varied, conf["light_vari"]) varied_imgname += 'l' if random.random() < conf["p_rotate"]: # todo change pos fn img_varied_ = random_rotate(img_varied, conf["rotate_angle_vari"]) if img_varied_.shape[0] >= conf["img_resize"] and img_varied_.shape[1] >= conf["img_resize"]: img_varied = img_varied_ varied_imgname += 'r' if random.random() < conf["p_horizonal_flip"]: # todo change pos fn img_varied = cv2.flip(img_varied, 1) varied_imgname += 'h' if random.random() < conf["p_vertical_flip"]: # todo change pos fn img_varied = cv2.flip(img_varied, 0) varied_imgname += 'v' output_file_path = os.path.join(prep_dir, '{}{}'.format(varied_imgname, ext)) cv2.imwrite(output_file_path, img_varied) else: output_file_path = os.path.join(conf["prep_test_dir"], '{}{}'.format(img_name, ext)) cv2.imwrite(output_file_path, img)
def operation(self, output, seed_file, resource_file=None): if resource_file == None: resource_file = self.__resource__["gif"] if output['filename'] != None and len(output['filename']) > 0: filename = output['filename'] else: filename = utils.extract_filename(seed_file) output['filename'] = filename + '_M1GIF' output['content'] = utils.extract_content(resource_file)[:1024] + \ output['content']
def operation(self, output, seed_file, resource_file=None): if resource_file == None: resource_file = self.__resource__["jpg"] if output['filename'] != None and len(output['filename']) > 0: filename = output['filename'] else: filename = utils.extract_filename(seed_file) output['filename'] = filename + '_M3JPG' output['filetype'] = utils.extract_filetype(resource_file)
def operation(self, output, seed_file, resource_file=None): if output['filename'] != None and len(output['filename']) > 0: filename = output['filename'] else: filename = utils.extract_filename(seed_file) output['filename'] = filename + '_M4PNG' if '.' not in output['fileext']: output['fileext'] = 'png' else: output['fileext'] = output['fileext'].rsplit('.', 1)[0] + '.png'
def process_dir(self, src_filepath: str, context_dotted_name: str, dirpath: str): for entry in utils.get_directory_contents(dirpath): entrypath = "{dirpath}/{entryname}".format(dirpath=dirpath, entryname=entry.name) if utils.is_valid_module(entrypath): entryname = utils.extract_filename(entry.name) module_name = "{context_dotted_name}.{entryname}".format( context_dotted_name=context_dotted_name, entryname=entryname) node = Node(name=module_name, ID=entrypath) self.graph[src_filepath].add(node) self.process_file(entrypath)
def _addremove_cake(self): """ add / remove cake no signal to update_graph """ if not self.widget.checkBox_ShowCake.isChecked(): return True if not self.model.poni_exist(): QtWidgets.QMessageBox.warning( self.widget, 'Warning', 'Choose PONI file first.') self.widget.checkBox_ShowCake.setChecked(False), return False # check if model.poni exist if not os.path.exists(self.model.poni): QtWidgets.QMessageBox.warning( self.widget, 'Warning', 'The poni does not exist in the path.') self.widget.checkBox_ShowCake.setChecked(False), return False if not self.model.base_ptn_exist(): QtWidgets.QMessageBox.warning( self.widget, 'Warning', 'Choose CHI file first.') self.widget.checkBox_ShowCake.setChecked(False) return False # check if model.poni is in temp_dir temp_dir = get_temp_dir(self.model.get_base_ptn_filename()) poni_filen = extract_filename(self.model.poni) + '.' + \ extract_extension(self.model.poni) if self.model.poni != os.path.join(temp_dir, poni_filen): shutil.copy(self.model.poni, os.path.join(temp_dir, poni_filen)) self.model.poni = os.path.join(temp_dir, poni_filen) self.widget.lineEdit_PONI.setText(self.model.poni) filen_tif = self.model.make_filename('tif', original=True) filen_tiff = self.model.make_filename('tiff', original=True) filen_mar3450 = self.model.make_filename('mar3450', original=True) filen_cbf = self.model.make_filename('cbf', original=True) if not ((os.path.exists(filen_tif) or os.path.exists(filen_tiff) or os.path.exists(filen_mar3450)) or os.path.exists(filen_cbf)): QtWidgets.QMessageBox.warning( self.widget, 'Warning', 'Cannot find image file: %s or %s or %s or %s.' % (filen_tif, filen_tiff, filen_mar3450, filen_cbf)) self.widget.checkBox_ShowCake.setChecked(False) return False if self.model.diff_img_exist() and \ self.model.same_filename_as_base_ptn( self.model.diff_img.img_filename): return True self.process_temp_cake() return True
def segment_transform(file_list, conf, mode): """Copy data from base XML and change to new one""" not_change_pos_keys = {"", "n", "l", "nl"} img_resize = conf["img_resize"] if mode == "train": org_dir = conf["train_data_dir"] prep_dir = conf["prep_train_dir"] else: org_dir = conf["val_data_dir"] prep_dir = conf["prep_val_dir"] for file_path in file_list: file_split = extract_filename(file_path) filename = file_split["filename"] img_name = file_split["imgname"] transform_type = img_name.split("_")[-1] if transform_type in not_change_pos_keys: org_xml_file = "_".join(img_name.split("_")[:-2]) + ".xml" new_xml_file = img_name + ".xml" org_xml_path = os.path.join(org_dir, org_xml_file) mytree, myroot = load_base_xml(org_xml_path) # FILE DESCRIBE CONFIG for p11 in myroot.iter('filename'): p11.text = filename for p12 in myroot.iter('path'): p12.text = os.path.join(prep_dir, org_xml_file) # IMAGE DATA CONFIG ratio_changed = 1 for p11 in myroot.iter('width'): ratio_changed = int(p11.text) / img_resize p11.text = str(img_resize) for p12 in myroot.iter('height'): p12.text = str(img_resize) for p1 in myroot.iter('xmin'): p1.text = str(float(p1.text) / ratio_changed) for p2 in myroot.iter('xmax'): p2.text = str(float(p2.text) / ratio_changed) for p3 in myroot.iter('ymin'): p3.text = str(float(p3.text) / ratio_changed) for p4 in myroot.iter('ymax'): p4.text = str(float(p4.text) / ratio_changed) else: # todo for position img changed case raise Exception("Error: Not implemented yet") new_xml_path = os.path.join(prep_dir, new_xml_file) write_base_cml(new_xml_path, mytree)
def operation(self, output, seed_file, resource_file=None): if output['filename'] != None and len(output['filename']) > 0: filename = output['filename'] else: filename = utils.extract_filename(seed_file) with open('./resource/test.jpg', 'rb') as fp: data = fp.read(8) output['filename'] = filename + '_M13' if output['content'][-1] == '\x0a': output['content'] = output['content'][:-1] + data else: output['content'] += data
def get_dataset_from_user(): global dataset_path global user_dataset_class_attribute while True: dataset_path = input('\nPlease provide the path of the dataset: ') if is_valid_path(dataset_path): user_dataset_class_attribute = input( '\nPlease provide the class label attribute name: ') return extract_filename(dataset_path) print("\nPath doesn't exist! Please provide a valid path!")
def operation(self, output, seed_file, resource_file=None): if output['filename'] != None and len(output['filename']) > 0: filename = output['filename'] else: filename = utils.extract_filename(seed_file) output['filename'] = filename + '_M11' tmp = "" for i in range(0, len(output['fileext'])): if i == '.': pass elif i % 2 == 0: tmp += output['fileext'][i].upper() else: tmp += output['fileext'][i] output['fileext'] = tmp
def makeMutatedData(self, mutate_list, seed_file, resource_file): output = { 'filename': utils.extract_filename(seed_file), 'fileext': utils.extract_fileext(seed_file), 'filetype': utils.extract_filetype(seed_file), 'content': utils.extract_content(seed_file) } # insert specific data for hash output['content'] = output['content'].replace( "%unique#", os.urandom(8).encode('hex')) for mutation in mutate_list: mutator = self.op_dict[mutation].mOP() mutator.operation(output, seed_file, resource_file) # XXX: Finally, use output variable to make request return output
def operation(self, output, seed_file, resource_file=None): base_headdata = """<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 96 105">\n""" base_taildata = """\n <g fill="#97C024" stroke="#97C024" stroke-linejoin="round" stroke-linecap="round"> <path d="M14,40v24M81,40v24M38,68v24M57,68v24M28,42v31h39v-31z" stroke-width="12"/> <path d="M32,5l5,10M64,5l-6,10 " stroke-width="2"/> </g> <path d="M22,35h51v10h-51zM22,33c0-31,51-31,51,0" fill="#97C024"/> <g fill="#FFF"> <circle cx="36" cy="22" r="2"/> <circle cx="59" cy="22" r="2"/> </g> </svg> """ output['content'] = base_headdata+output['content']+base_taildata #print output['content'] if output['filename'] != None and len(output['filename']) > 0: filename = output['filename'] else: filename = utils.extract_filename(seed_file) output['filename'] = filename + '_M8' output['fileext'] = 'svg'
def get_poni(self): """ Opens a pyFAI calibration file signal to update_graph """ filen = QtWidgets.QFileDialog.getOpenFileName( self.widget, "Open a PONI File", self.model.chi_path, "PONI files (*.poni)")[0] filename = str(filen) if os.path.exists(filename): # copy the chose file to temp_dir temp_dir = get_temp_dir(self.model.get_base_ptn_filename()) shutil.copy(filename, temp_dir) filen = extract_filename(filename) + '.' + \ extract_extension(filename) # set filename to that exists in temp_dir self.model.poni = os.path.join(temp_dir, filen) self.widget.lineEdit_PONI.setText(self.model.poni) if self.model.diff_img_exist(): self.produce_cake() self._apply_changes_to_graph()
def operation(self, output, seed_file, resource_file=None): base_data='''TESTEML Content-Type: text/html Content-Transfer-Encoding: quoted-printable ''' normalstr = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890 \n\t' emlenc = lambda x: '='+hex(ord(x))[2:] data = '' for i in output['content']: if i not in normalstr: data +=emlenc(i) else: data += i output['content'] = base_data+data if output['filename'] != None and len(output['filename']) > 0: filename = output['filename'] else: filename = utils.extract_filename(seed_file) #print output['content'] output['filename'] = filename + '_M6' output['fileext'] = 'eml'
def process_file(self, filepath: str): # Ensure that we don't revisit files if filepath in self.graph: return logging.info( "[DependencyAnalyzer::process_file] Processing File {file}.". format(file=filepath)) # Iterate through the parse tree and process any imports self.graph[filepath] = set() tree, lines = self.parser.parse_file(filepath) file = File(filepath, lines) imports = {} used_imports = set() for node in tree.root_node.children: if self.is_import(node): import_node_list = self.handle_import(file, node) for import_node in import_node_list: if import_node: imports[import_node.alias] = import_node elif self.config.mark_unused: token_used_imports = self.handle_unknown_token( file, node, imports) used_imports = used_imports.union(token_used_imports) if self.config.mark_unused and utils.extract_filename( filepath) != "__init__": for dependency in self.graph[filepath]: #print("skip", dependency.ID, filepath) alias = dependency.alias if alias and (alias in imports) and ( alias not in used_imports) and (dependency.ID == imports[alias].ID): print(filepath, alias, dependency.ID) dependency.labels = ["unused"]
def update(self): """ show a list of jcpds in the list window of tab 3 called from maincontroller """ n_columns = 4 n_rows = self.model.waterfall_ptn.__len__() # count for number of jcpds self.widget.tableWidget_wfPatterns.setColumnCount(n_columns) self.widget.tableWidget_wfPatterns.setRowCount(n_rows) self.widget.tableWidget_wfPatterns.horizontalHeader().setVisible(True) self.widget.tableWidget_wfPatterns.setHorizontalHeaderLabels( ['', '', '', 'Wavelength']) self.widget.tableWidget_wfPatterns.setVerticalHeaderLabels( [extract_filename(wfp.fname) for wfp in self.model.waterfall_ptn]) for row in range(n_rows): # column 0 - checkbox item0 = QtWidgets.QTableWidgetItem() item0.setFlags(QtCore.Qt.ItemIsUserCheckable | QtCore.Qt.ItemIsEnabled) if self.model.waterfall_ptn[row].display: item0.setCheckState(QtCore.Qt.Checked) else: item0.setCheckState(QtCore.Qt.Unchecked) self.widget.tableWidget_wfPatterns.setItem(row, 0, item0) # column 1 - color item2 = QtWidgets.QTableWidgetItem('') self.widget.tableWidget_wfPatterns.setItem(row, 1, item2) # column 3 - color setup self.widget.tableWidget_wfPatterns_pushButton_color = \ QtWidgets.QPushButton('.') self.widget.tableWidget_wfPatterns.item(row, 1).setBackground( QtGui.QColor(self.model.waterfall_ptn[row].color)) self.widget.tableWidget_wfPatterns_pushButton_color.clicked.\ connect(self._handle_ColorButtonClicked) self.widget.tableWidget_wfPatterns.setCellWidget( row, 2, self.widget.tableWidget_wfPatterns_pushButton_color) # column 3 - wavelength self.widget.tableWidget_wfPatterns_doubleSpinBox_wavelength = \ QtWidgets.QDoubleSpinBox() self.widget.tableWidget_wfPatterns_doubleSpinBox_wavelength.\ setAlignment( QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.widget.tableWidget_wfPatterns_doubleSpinBox_wavelength.\ setMaximum(2.0) self.widget.tableWidget_wfPatterns_doubleSpinBox_wavelength.\ setSingleStep(0.0001) self.widget.tableWidget_wfPatterns_doubleSpinBox_wavelength.\ setDecimals(4) self.widget.tableWidget_wfPatterns_doubleSpinBox_wavelength.\ setProperty("value", self.model.waterfall_ptn[row].wavelength) self.widget.tableWidget_wfPatterns_doubleSpinBox_wavelength.\ valueChanged.connect( self._handle_doubleSpinBoxChanged) self.widget.tableWidget_wfPatterns_doubleSpinBox_wavelength.\ setStyle(SpinBoxFixStyle()) self.widget.tableWidget_wfPatterns.setCellWidget( row, 3, self.widget.tableWidget_wfPatterns_doubleSpinBox_wavelength) self.widget.tableWidget_wfPatterns_doubleSpinBox_wavelength.\ setKeyboardTracking(False) self.widget.tableWidget_wfPatterns_doubleSpinBox_wavelength.\ setFocusPolicy(QtCore.Qt.StrongFocus) self.widget.tableWidget_wfPatterns.resizeColumnsToContents() # self.widget.tableWidget_wfPatterns.resizeRowsToContents() self.widget.tableWidget_wfPatterns.itemClicked.connect( self._handle_ItemClicked)
def report_on(g_dir, g_file): g = nx.read_graphml(os.path.join(g_dir, g_file)) print('%s,%d,%d,%d' % (utils.extract_filename(g_file), g.number_of_nodes(), g.number_of_edges(), nx.number_connected_components(g)))