def from_directory(cls, dirname): print "Loading from directory: ", dirname filenames = glob.glob(dirname+"/*.fits") + glob.glob(dirname+"/*.fits.gz") print 'got %d files' % len(filenames) cat_name=dirname.strip(os.path.sep).split(os.path.sep)[-1] cat = cls.from_multiple_fits(filenames, cat_name) return cat
def cleanupFiles(): # First get rid of modified files for l in ["l1", "l2", "l3"]: arcpy.Delete_management(l) for f in glob.glob("C:\\Arctmp\\*"): try: shutil.rmtree(f) except: print "UNABLE TO REMOVE:", f # Now remove the old directory for i in xrange(0, 1000000): new_workspace = "C:\\Arctmp\\workspace." + str(i) if not os.path.exists(new_workspace): break print "TESTING USING WORKSPACE", new_workspace # Now move in fresh copies shutil.copytree("C:\\Arcbase", new_workspace) print "CONTENTS:" arcpy.env.workspace = new_workspace for f in sorted(glob.glob(arcpy.env.workspace + "\\*.shp")): print f for f in sorted(glob.glob(arcpy.env.workspace + "\\*.lyr")): print f for f in sorted(glob.glob(arcpy.env.workspace + "\\*.gdb")): print f
def _load_imdb(self): dir_path = os.path.join(config.data_path, self.folder) data_path = os.path.join(dir_path, self.which_set) pos_path = os.path.join(data_path, 'pos') neg_path = os.path.join(data_path, 'neg') files = glob.glob(pos_path+'/*.txt') pos_strings = [open(f, 'r').read() for f in files] pos_labels = np.ones(len(files)) files = glob.glob(neg_path+'/*.txt') neg_strings = [open(f, 'r').read() for f in files] neg_labels = np.zeros(len(files)) targets = np.hstack((pos_labels, neg_labels)) targets = numpy.array(targets, dtype='int32').reshape((-1, 1)) features = np.array(pos_strings + neg_strings) #n = 25000 / 2 #features = features[n-1000:n+1000] #targets = targets[n-1000:n+1000] self.num_examples = len(features) if self.sorted == True: index = np.vectorize(len)(features).argsort() features = features[index] targets = targets[index] return (features, targets)
def _InstrumentExecutables(self): build_dir = self._build_dir work_dir = self._work_dir _LOGGER.info('Build dir "%s".', build_dir) # Make a copy of all unittest executables, DLLs, PDBs and test_data in # the build directory. for pattern in ('*_unittests.exe', '*.dll', '*.pdb', 'test_data'): files = glob.glob(os.path.join(build_dir, pattern)) for path in files: _LOGGER.info('Copying "%s" to "%s".', path, work_dir) if os.path.isdir(path): # If the source file is a directory, do a recursive copy. dst = os.path.join(work_dir, os.path.basename(path)) shutil.copytree(path, dst) else: shutil.copy(path, work_dir) # Instrument all EXEs in the work dir. for exe in glob.glob(os.path.join(work_dir, '*.exe')): self._InstrumentOneFile(exe) # And the DLLs we've specified. for dll in _DLLS_TO_INSTRUMENT: self._InstrumentOneFile(os.path.join(work_dir, dll))
def test_onset_functions(): # Load in all files in the same order ref_files = sorted(glob.glob(REF_GLOB)) est_files = sorted(glob.glob(EST_GLOB)) sco_files = sorted(glob.glob(SCORES_GLOB)) assert len(ref_files) == len(est_files) == len(sco_files) > 0 # Unit tests for metric in [mir_eval.onset.f_measure]: yield (__unit_test_onset_function, metric) # Regression tests for ref_f, est_f, sco_f in zip(ref_files, est_files, sco_files): with open(sco_f, 'r') as f: expected_scores = json.load(f) # Load in an example onset annotation reference_onsets = mir_eval.io.load_events(ref_f) # Load in an example onset tracker output estimated_onsets = mir_eval.io.load_events(est_f) # Compute scores scores = mir_eval.onset.evaluate(reference_onsets, estimated_onsets) # Compare them for metric in scores: # This is a simple hack to make nosetest's messages more useful yield (__check_score, sco_f, metric, scores[metric], expected_scores[metric])
def write_csv_files(self, overwrite=False): self.extract_images() for setn in ('train', 'val'): img_dir = os.path.join(self.out_dir, setn) csvfile = getattr(self, setn + '_file') neon_logger.display("Getting %s file list" % (setn)) if os.path.exists(csvfile) and not overwrite: neon_logger.display("File %s exists, not overwriting" % (csvfile)) continue flines = [] subdirs = glob(os.path.join(img_dir, '*')) for subdir in subdirs: subdir_label = os.path.basename(subdir) # This is the int label files = glob(os.path.join(subdir, self.file_pattern)) flines += [(filename, subdir_label) for filename in files] if setn == 'train': np.random.seed(0) np.random.shuffle(flines) with gzip.open(csvfile, 'wb') as f: f.write('filename,l_id\n') for tup in flines: f.write('{},{}\n'.format(*tup))
def get_cached_file(self, file_path, file_glob, required=False, use_first=False, **addl_args): if file_glob == None: return None if hasattr(file_glob, '__iter__'): found_files = [] for curr_glob in file_glob: curr_files = glob.glob( os.path.join(file_path, curr_glob) ) if len(curr_files) > 0: found_files = curr_files break else: found_files = glob.glob( os.path.join(file_path, file_glob) ) if len(found_files) == 0: if required: raise OSError('Could not find at path: "%s" any files matching glob: "%s"' % (file_path, file_glob)) else: return None elif len(found_files) > 1 and not use_first: raise OSError('Found too many files files at path: "%s" with glob: "%s", found: %s' % (file_path, file_glob, found_files)) if self.file_cache.has_key(found_files[0]): file_obj = self.file_cache[ found_files[0] ] else: file_obj = OCO_Matrix( found_files[0], **addl_args ) self.file_cache[ found_files[0] ] = file_obj return file_obj
def test_devtool_reset_all(self): # Check preconditions workspacedir = os.path.join(self.builddir, 'workspace') self.assertTrue(not os.path.exists(workspacedir), 'This test cannot be run with a workspace directory under the build directory') tempdir = tempfile.mkdtemp(prefix='devtoolqa') self.track_for_cleanup(tempdir) self.track_for_cleanup(workspacedir) self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') testrecipe1 = 'mdadm' testrecipe2 = 'cronie' result = runCmd('devtool modify -x %s %s' % (testrecipe1, os.path.join(tempdir, testrecipe1))) result = runCmd('devtool modify -x %s %s' % (testrecipe2, os.path.join(tempdir, testrecipe2))) result = runCmd('devtool build %s' % testrecipe1) result = runCmd('devtool build %s' % testrecipe2) stampprefix1 = get_bb_var('STAMP', testrecipe1) self.assertTrue(stampprefix1, 'Unable to get STAMP value for recipe %s' % testrecipe1) stampprefix2 = get_bb_var('STAMP', testrecipe2) self.assertTrue(stampprefix2, 'Unable to get STAMP value for recipe %s' % testrecipe2) result = runCmd('devtool reset -a') self.assertIn(testrecipe1, result.output) self.assertIn(testrecipe2, result.output) result = runCmd('devtool status') self.assertNotIn(testrecipe1, result.output) self.assertNotIn(testrecipe2, result.output) matches1 = glob.glob(stampprefix1 + '*') self.assertFalse(matches1, 'Stamp files exist for recipe %s that should have been cleaned' % testrecipe1) matches2 = glob.glob(stampprefix2 + '*') self.assertFalse(matches2, 'Stamp files exist for recipe %s that should have been cleaned' % testrecipe2)
def annotate(self, conf, subject): subject = dict(subject) lib_jars = list() for lib_dir in subject['lib_dirs']: lib_jars += glob.glob(os.path.join(lib_dir, '*.jar')) lib_jars += glob.glob(os.path.join(lib_dir, '*.aar')) subject['lib_jars'] = lib_jars subject['soot_classpath'] = ':'.join(( ':'.join(subject['class_dirs']), ':'.join(subject['lib_jars']), subject['classpath'], )) targets = list() if 'target' in subject: targets.append(subject['target']) else: targets.extend(subject['targets']) subject['jpdg_cmd'] = [ 'java', '-Xmx8g', '-jar', self.jpdg_jar, '-c', subject['soot_classpath'], '-l', 'op', ] for t in targets: subject['jpdg_cmd'] += ['-d', t] for ex_dir in subject['exclude_pkgs']: subject['jpdg_cmd'] += ['-e', ex_dir] return subject
def main(): files = glob.glob("./scans/*.jpg") files += glob.glob("./scans/*.jpeg") for f in files: reset_stats() print "Processing: " + f.split("/")[len(f.split("/")) - 1] schedule = Schedule() schedule.load_data() if schedule.get_has_schedule(): scan_image(f, schedule) print "Sheet ok? ", while True: cv2.imshow("image", cv2.resize(img, (446, 578))) cv2.moveWindow("image", 0, 0) # user_in = raw_input() key = cv2.waitKey(-1) if key == ord("y"): print "Sheet ok... Dumping data" dump_stats() os.remove(f) break elif key == ord("n"): print "Marking to redo" #os.rename(f, "./scans/redo/" + f.split("/")[len(f.split("/")) - 1]) break elif key == ord("q"): exit(0) else: continue cv2.destroyAllWindows() else: print "Unable to load schedule... Aborting"
def test_stage_package_gets_cached(self): self.run_snapcraft(["pull", "oneflatwithstagepackages"], "dump") # Verify the 'hello' deb package was cached. cache_dir = os.path.join( xdg.BaseDirectory.xdg_cache_home, "snapcraft", "stage-packages", "apt" ) archive_dir = os.path.join("var", "cache", "apt", "archives") cached = glob.glob(os.path.join(cache_dir, "*", archive_dir, "hello*")) self.assertThat(cached, HasLength(1)) cached_deb = cached[0] staged = glob.glob( os.path.join( "parts", "oneflatwithstagepackages", "ubuntu", "download", "hello*" ) ) self.assertThat(staged, HasLength(1)) staged_deb = staged[0] # Verify that the staged and cached debs are the same file (hard # linked) by comparing inodes. cached_deb_inode = os.stat(cached_deb).st_ino self.assertThat(cached_deb_inode, Equals(os.stat(staged_deb).st_ino)) # Now clean the part and pull again. self.run_snapcraft("clean", "dump") self.run_snapcraft(["pull", "oneflatwithstagepackages"], "dump") # Verify that the staged deb is _still_ the same one from the cache. self.assertThat(cached_deb_inode, Equals(os.stat(staged_deb).st_ino))
def generate_rst(): """generate chX.rst in current working directory""" cwd = os.getcwd() demo_dir = os.path.join(cwd, 'demos') chapters = os.listdir(demo_dir) for chapter in chapters: if not os.path.isdir(os.path.join(demo_dir, chapter)): continue reg_py = os.path.join(demo_dir, chapter, '*.py') scripts = glob.glob(reg_py) rst_file = chapter + '.rst' rst_file = os.path.join(demo_dir, chapter, rst_file) with open(rst_file, 'w') as f: f.write(chapter) f.write('\n========================================\n') for script in scripts: script_name = os.path.basename(script) f.write('\n' + script_name[:-3]) f.write('\n----------------------------------------\n') reg_png = os.path.join(demo_dir, chapter, script_name[:-3] + '*.png') for img in glob.glob(reg_png): img_name = os.path.basename(img) f.write(".. image:: " + img_name + "\n") f.write(".. literalinclude:: " + script_name + "\n")
def _file_configs_paths(osname, agentConfig): """ Retrieve all the file configs and return their paths """ try: confd_path = get_confd_path(osname) all_file_configs = glob.glob(os.path.join(confd_path, '*.yaml')) all_default_configs = glob.glob(os.path.join(confd_path, '*.yaml.default')) except PathNotFound as e: log.error("No conf.d folder found at '%s' or in the directory where the Agent is currently deployed.\n" % e.args[0]) sys.exit(3) if all_default_configs: current_configs = set([_conf_path_to_check_name(conf) for conf in all_file_configs]) for default_config in all_default_configs: if not _conf_path_to_check_name(default_config) in current_configs: all_file_configs.append(default_config) # Compatibility code for the Nagios checks if it's still configured # in datadog.conf # FIXME: 6.x, should be removed if not any('nagios' in config for config in itertools.chain(*all_file_configs)): # check if it's configured in datadog.conf the old way if any([nagios_key in agentConfig for nagios_key in NAGIOS_OLD_CONF_KEYS]): all_file_configs.append('deprecated/nagios') return all_file_configs
def strip_symbols(): bin_dir = os.path.join(conf[CONF_BUILDDIR], 'pack', 'bin') ignored = [] def do_strip(fn): run('strip "%s"' % fn) info('stripping: %s' % fn) def should_ignore(path): '''Do not strip python.dll and msvc*.dll ''' name = os.path.basename(path).lower() return name.startswith('python') or name.startswith('msvc') for dll in glob.glob(os.path.join(bin_dir, '*.dll')): if should_ignore(dll): ignored.append(dll) else: do_strip(dll) for exe in glob.glob(os.path.join(bin_dir, '*.exe')): do_strip(exe) info('----------------------------') info('ignored:') for name in ignored: info('>> %s' % name)
def main(args): # Default invocation will verify the golden files are unchanged. failed = 0 if not args: args = ['--wnone', '--diff', '--test', '--dstroot=.'] ParseOptions(args) idldir = os.path.split(sys.argv[0])[0] idldir = os.path.join(idldir, 'test_cgen', '*.idl') filenames = glob.glob(idldir) ast = ParseFiles(filenames) if hgen.GenerateRelease(ast, 'M14', {}): print "Golden file for M14 failed." failed = 1 else: print "Golden file for M14 passed." idldir = os.path.split(sys.argv[0])[0] idldir = os.path.join(idldir, 'test_cgen_range', '*.idl') filenames = glob.glob(idldir) ast = ParseFiles(filenames) if hgen.GenerateRange(ast, ['M13', 'M14', 'M15', 'M16', 'M17'], {}): print "Golden file for M13-M17 failed." failed =1 else: print "Golden file for M13-M17 passed." return failed
def setRepositoryRevisions(self): # expand possible environment variables in paths if isinstance(self._args.repo_scan_base_dirs, basestring): self._args.repo_scan_base_dirs = [self._args.repo_scan_base_dirs] self._args.repo_scan_base_dirs = [os.path.expandvars(repoScanBaseDir) for repoScanBaseDir in self._args.repo_scan_base_dirs] # construct possible scan paths subDirWildcards = ["*/" * level for level in range(self._args.repo_scan_depth+1)] scanDirWildcards = [os.path.join(repoScanBaseDir, subDirWildcard) for repoScanBaseDir in self._args.repo_scan_base_dirs for subDirWildcard in subDirWildcards] # globbing and filter for directories scanDirs = tools.flattenList([glob.glob(scanDirWildcard) for scanDirWildcard in scanDirWildcards]) scanDirs = [scanDir for scanDir in scanDirs if os.path.isdir(scanDir)] # key: directory to check type of repository # value: command to extract the revision repoVersionCommands = { ".git" : "git rev-parse HEAD", ".svn" : "svn info"# | grep Revision | awk '{print $2}'" } # loop over dirs and revision control systems and write revisions to the config dict for repoDir, currentRevisionCommand in repoVersionCommands.items(): repoScanDirs = [os.path.join(scanDir, repoDir) for scanDir in scanDirs] repoScanDirs = [glob.glob(os.path.join(scanDir, repoDir)) for scanDir in scanDirs] repoScanDirs = tools.flattenList([glob.glob(os.path.join(scanDir, repoDir)) for scanDir in scanDirs]) repoScanDirs = [os.path.abspath(os.path.join(repoScanDir, "..")) for repoScanDir in repoScanDirs] for repoScanDir in repoScanDirs: popenCout, popenCerr = subprocess.Popen(currentRevisionCommand.split(), stdout=subprocess.PIPE, cwd=repoScanDir).communicate() self._config[repoScanDir] = popenCout.replace("\n", "")
def lookup_copyright_notice(self, ufo_folder): current_path = ufo_folder try: contents = open(os.path.join(ufo_folder, 'fontinfo.plist')).read() copyright = self.grep_copyright_notice(contents) if copyright: return copyright except (IOError, OSError): pass while os.path.realpath(self.operator.path) != current_path: # look for all text files inside folder # read contents from them and compare with copyright notice # pattern files = glob.glob(os.path.join(current_path, '*.txt')) files += glob.glob(os.path.join(current_path, '*.ttx')) for filename in files: with open(os.path.join(current_path, filename)) as fp: match = COPYRIGHT_REGEX.search(fp.read()) if not match: continue return match.group(0).strip(',\r\n') current_path = os.path.join(current_path, '..') # go up current_path = os.path.realpath(current_path) return
def findPR650(ports=None): """DEPRECATED (as of v.1.60.01). Use :func:`psychopy.hardware.findPhotometer()` instead, which finds a wider range of devices """ logging.error("DEPRECATED (as of v.1.60.01). Use psychopy.hardware.findPhotometer() instead, which "\ +"finds a wider range of devices") if ports==None: if sys.platform=='darwin': ports=[] #try some known entries in /dev/tty. used by keyspan ports.extend(glob.glob('/dev/tty.USA*'))#keyspan twin adapter is usually USA28X13P1.1 ports.extend(glob.glob('/dev/tty.Key*'))#some are Keyspan.1 or Keyserial.1 ports.extend(glob.glob('/dev/tty.modem*'))#some are Keyspan.1 or Keyserial.1 if len(ports)==0: logging.error("couldn't find likely serial port in /dev/tty.* Check for \ serial port name manually, check drivers installed etc...") elif sys.platform=='win32': ports = range(11) elif type(ports) in [int,float]: ports=[ports] #so that we can iterate pr650=None logging.info('scanning serial ports...\n\t') logging.console.flush() for thisPort in ports: logging.info(str(thisPort)); logging.console.flush() pr650 = Photometer(port=thisPort, meterType="PR650", verbose=False) if pr650.OK: logging.info(' ...OK\n'); logging.console.flush() break else: pr650=None logging.info('...Nope!\n\t'); logging.console.flush() return pr650
def repackage_archive_zip_to_pmc_zip(self, doi_id): "repackage the zip file in the TMP_DIR to a PMC zip format" # unzip contents zip_input_dir = os.path.join(self.get_tmp_dir(), self.TMP_DIR) zip_extracted_dir = os.path.join(self.get_tmp_dir(), self.JUNK_DIR) zip_renamed_files_dir = os.path.join(self.get_tmp_dir(), self.RENAME_DIR) pmc_zip_output_dir = os.path.join(self.get_tmp_dir(), self.INPUT_DIR) archive_zip_name = glob.glob(zip_input_dir + "/*.zip")[0] with zipfile.ZipFile(archive_zip_name, 'r') as myzip: myzip.extractall(zip_extracted_dir) # rename the files and profile the files file_name_map = article_processing.rename_files_remove_version_number( files_dir = zip_extracted_dir, output_dir = zip_renamed_files_dir ) if self.logger: self.logger.info("FTPArticle running %s workflow for article %s, file_name_map" % (self.workflow, self.doi_id)) self.logger.info(file_name_map) # convert the XML article_xml_file = glob.glob(zip_renamed_files_dir + "/*.xml")[0] article_processing.convert_xml(xml_file=article_xml_file, file_name_map=file_name_map) # rezip the files into PMC zip format soup = parser.parse_document(article_xml_file) volume = parser.volume(soup) pmc_zip_file_name = article_processing.new_pmc_zip_filename(self.journal, volume, doi_id) with zipfile.ZipFile(os.path.join(pmc_zip_output_dir, pmc_zip_file_name), 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as new_zipfile: dirfiles = article_processing.file_list(zip_renamed_files_dir) for df in dirfiles: filename = df.split(os.sep)[-1] new_zipfile.write(df, filename) return True
def install_new_policy(projdir, tmpdir): """ Copies the polkit policy files. """ files = glob.glob(projdir + '/data/*.policy') + \ glob.glob(projdir + '/modules/*/data/*.policy') return _copy_files(files, '/usr/share/polkit-1/actions/', tmpdir)
def getPurchaseReport(): purchaseFiles = [] for purchases in glob.glob('purchases/purchase*'): purchaseFiles.append(purchases) for purchases in glob.glob('purchases/Item*'): itemlist = purchases item_dict = {} with open(itemlist,'r') as itemfile: all_lines = itemfile.readlines() for i in range(2,len(all_lines)): item = all_lines[i].split() item_dict.update({item[0]:item[1]}) #print item_dict report_dict = {} for i in range(0,len(purchaseFiles)): with open(purchaseFiles[i],'r') as purchasefile: item_total = 0 purchase_list = purchasefile.readlines() #print purchaseFiles[i][19:22] for j in range(2,len(purchase_list)): purchase = purchase_list[j].split() #print float(item_dict[purchase[0]][1:]) #print float(purchase[1]) item_total = float(item_dict[purchase[0]][1:])*float(purchase[1])+item_total #print item_total report_dict.update({int(purchaseFiles[i][21]):"{0:.2f}".format(item_total)}) #print purchaseFiles[i][19:22] print report_dict return report_dict
def copy_build_directories_vs(dist_build, build_dir): """Copy the build/visual-studio directories to the distribution directory. """ buildfiles = __astyle_dir + "/build/" # copy solution files vsdir = '/' + build_dir + '/' dist_astyle_vs20xx = dist_build + vsdir os.mkdir(dist_astyle_vs20xx) slnfiles = glob.glob(buildfiles + vsdir + "*.sln") for sln in slnfiles: shutil.copy(sln, dist_astyle_vs20xx) # build project directories for projdir in ("/AStyle/", "/AStyle Dll/", "/AStyle Java/", "/AStyle Lib/"): dist_astyle_proj = dist_astyle_vs20xx[:-1] + projdir os.mkdir(dist_astyle_proj) # copy project files projfiles = glob.glob(buildfiles + vsdir[:-1] + projdir + "*.*proj") files_copied = 0 for proj in projfiles: files_copied += 1 shutil.copy(proj, dist_astyle_proj) if vsdir[1:-1] >= "vs2010": filtfiles = glob.glob(buildfiles + vsdir[:-1] + projdir + "*.*.filters") for filter_in in filtfiles: files_copied += 1 shutil.copy(filter_in, dist_astyle_proj) # verify number of files copied if files_copied != 2: libastyle.system_exit("Error in number of build files copied: " + str(files_copied))
def get_batches_fn(batch_size): """ Create batches of training data :param batch_size: Batch Size :return: Batches of training data """ image_paths = glob(os.path.join(data_folder, 'image_2', '*.png')) label_paths = { re.sub(r'_(lane|road)_', '_', os.path.basename(path)): path for path in glob(os.path.join(data_folder, 'gt_image_2', '*_road_*.png'))} background_color = np.array([255, 0, 0]) random.shuffle(image_paths) for batch_i in range(0, len(image_paths), batch_size): images = [] gt_images = [] for image_file in image_paths[batch_i:batch_i+batch_size]: gt_image_file = label_paths[os.path.basename(image_file)] image = scipy.misc.imresize(scipy.misc.imread(image_file), image_shape) gt_image = scipy.misc.imresize(scipy.misc.imread(gt_image_file), image_shape) gt_bg = np.all(gt_image == background_color, axis=2) gt_bg = gt_bg.reshape(*gt_bg.shape, 1) gt_image = np.concatenate((gt_bg, np.invert(gt_bg)), axis=2) images.append(image) gt_images.append(gt_image) yield np.array(images), np.array(gt_images)
def get_tests_info(input_dir, msg_dir, prefix, suffix): """get python input examples and output messages We use following conventions for input files and messages: for different inputs: test for python >= x.y -> input = <name>_pyxy.py test for python < x.y -> input = <name>_py_xy.py for one input and different messages: message for python >= x.y -> message = <name>_pyxy.txt lower versions -> message with highest num """ result = [] for fname in glob(join(input_dir, prefix + '*' + suffix)): infile = basename(fname) fbase = splitext(infile)[0] # filter input files : pyrestr = fbase.rsplit('_py', 1)[-1] # like _26 or 26 if pyrestr.isdigit(): # '24', '25'... if SYS_VERS_STR < pyrestr: continue if pyrestr.startswith('_') and pyrestr[1:].isdigit(): # skip test for higher python versions if SYS_VERS_STR >= pyrestr[1:]: continue messages = glob(join(msg_dir, fbase + '*.txt')) # the last one will be without ext, i.e. for all or upper versions: if messages: for outfile in sorted(messages, reverse=True): py_rest = outfile.rsplit('_py', 1)[-1][:-4] if py_rest.isdigit() and SYS_VERS_STR >= py_rest: break else: outfile = None result.append((infile, outfile)) return result
def deploy(self, file, contextroot=None, deploymentorder=100, libraries=[]): files = glob.glob(file) if len(files) != 1: abort("Exactly one file must match " + file) cmd = self.asadminCommand + " " + "deploy" if self.version >= 4: cmd = cmd + " --deploymentorder " + str(deploymentorder) if contextroot: cmd = cmd + " --contextroot " + contextroot if libraries: libstring = "" for library in libraries: path = os.path.join(self.lib_path, library) libs = glob.glob(path) if len(libs) != 1: abort("Exactly one library must match " + path) libadd = os.path.basename(libs[0]) if libstring: libstring += "," + libadd else: libstring = "--libraries " + libadd cmd = cmd + " " + libstring cmd = cmd + " " + files[0] if self.verbosity: print "\nexecute: " + cmd out, err, rc = self.execute(cmd) if self.verbosity > 1: if out: print out if err: for line in err.splitlines(): line = line.strip() if line: if line.startswith("PER01"): continue print line
def init_notebooks(self): """Construct the list of notebooks. If notebooks are passed on the command-line, they override notebooks specified in config files. Glob each notebook to replace notebook patterns with filenames. """ # Specifying notebooks on the command-line overrides (rather than adds) # the notebook list if self.extra_args: patterns = self.extra_args else: patterns = self.notebooks # Use glob to replace all the notebook patterns with filenames. filenames = [] for pattern in patterns: # Use glob to find matching filenames. Allow the user to convert # notebooks without having to type the extension. globbed_files = glob.glob(pattern) globbed_files.extend(glob.glob(pattern + '.ipynb')) if not globbed_files: self.log.warn("pattern %r matched no files", pattern) for filename in globbed_files: if not filename in filenames: filenames.append(filename) self.notebooks = filenames
def w2p_pack_plugin(filename, path, plugin_name): """Packs the given plugin into a w2p file. Will match files at:: <path>/*/plugin_[name].* <path>/*/plugin_[name]/* """ filename = abspath(filename) path = abspath(path) if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name): raise Exception("Not a web2py plugin name") plugin_tarball = tarfile.open(filename, 'w:gz') try: app_dir = path while app_dir[-1] == '/': app_dir = app_dir[:-1] files1 = glob.glob( os.path.join(app_dir, '*/plugin_%s.*' % plugin_name)) files2 = glob.glob( os.path.join(app_dir, '*/plugin_%s/*' % plugin_name)) for file in files1 + files2: plugin_tarball.add(file, arcname=file[len(app_dir) + 1:]) finally: plugin_tarball.close()
def process_all_input(self, projectname, path, queue, \ increment_preprocessed_func, increment_processed_func, cancelled_func) : dir_re = re.compile(".*c(\d+)$") input_re = re.compile("^datain_(\d+)\..*") listing = filter(lambda x : os.path.isdir(x) and dir_re.match(x), glob.glob(path + os.sep + "*")) if len(listing) == 0 : raise PluginError("no chromosome directories to process in %s" % path) for dir in listing : chromo = dir_re.match(dir).group(1) inputfiles = glob.glob(dir + os.sep + 'datain_*') for f in inputfiles : if cancelled_func() : return dirname,filename = os.path.split(f) m = input_re.match(filename) if not m : continue fragid = m.group(1) if os.path.exists(dirname + os.sep + ("gh_%s.out" % fragid)) : increment_processed_func() continue input = map(lambda x : dirname + os.sep + (x % (fragid,chromo)), \ ['datain_%s.%s','map_%s.%s','pedin_%s.%s','setup_%s.%s']) output = dirname + os.sep + ("gh_%s.out" % fragid) tmp = (input,output) queue.put( tmp ) increment_preprocessed_func()
def config(): # spark-default.conf spark_defaults_tmp_location = os.path.join(tmp_dir,"spark-defaults.conf") spark_default_final_location = os.path.join(spark_home,"conf") with open(spark_defaults_tmp_location,'a') as spark_defaults: spark_defaults.write("spark.eventLog.enabled true\n") spark_defaults.write("spark.eventLog.dir {0}\n".format(spark_evlogs)) subprocess.check_call(["/bin/mv",spark_defaults_tmp_location,spark_default_final_location]) # bashrc file with open("/home/hadoop/.bashrc","a") as bashrc: bashrc.write("export SCALA_HOME={0}".format(scala_home)) # spark-env.sh spark_env_tmp_location = os.path.join(tmp_dir,"spark-env.sh") spark_env_final_location = os.path.join(spark_home,"conf") files= glob.glob("{0}/{1}/share/*/*/*/hadoop-*lzo.jar".format(hadoop_apps,hadoop_version)) if len(files) < 1: files=glob.glob("{0}/{1}/share/*/*/*/hadoop-*lzo-*.jar".format(hadoop_apps,hadoop_version)) if len(files) < 1: print "lzo not found inside {0}/{1}/share/".format(hadoop_apps,hadoop_version) else: lzo_jar=files[0] #subprocess.check_call(["/bin/mkdir","-p",spark_log_dir]) subprocess.call(["/bin/mkdir","-p",spark_log_dir]) with open(spark_env_tmp_location,'a') as spark_env: spark_env.write("export SPARK_DAEMON_JAVA_OPTS=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps\"\n") spark_env.write("export SPARK_LOCAL_DIRS={0}\n".format(local_dir)) spark_env.write("export SPARK_LOG_DIR={0}\n".format(spark_log_dir)) spark_env.write("export SPARK_CLASSPATH=\"{0}/emr/*:{1}/emrfs/*:{2}/share/hadoop/common/lib/*:{3}:/home/hadoop/hive/conf/*\"\n".format(spark_classpath,spark_classpath,hadoop_home,lzo_jar)) subprocess.check_call(["mv",spark_env_tmp_location,spark_env_final_location])
def complete(self, txt): """ Returns the next completion for txt, or None if there is no completion. """ if not self.lookup: self.lookup = [] if txt == "" or txt[0] not in "~/": txt = "~/" + txt path = os.path.expanduser(txt) if os.path.isdir(path): files = glob.glob(os.path.join(path, "*")) prefix = txt else: files = glob.glob(path + "*") prefix = os.path.dirname(txt) prefix = prefix.rstrip("/") or "/" for f in files: display = os.path.join(prefix, os.path.basename(f)) if os.path.isdir(f): display += "/" self.lookup.append((display, f)) self.lookup.sort() self.offset = -1 self.lookup.append((txt, txt)) self.offset += 1 if self.offset >= len(self.lookup): self.offset = 0 ret = self.lookup[self.offset] self.thisfinal = ret[1] return ret[0]
import cv2 import glob import time import numpy as np images = glob.glob("/home/pi/Desktop/cone/real/*.jpg") #print images images.sort() tmp_list = images[0].split('/') title = tmp_list[-1] image = cv2.imread(images[0]) height, width, layers = image.shape video = cv2.VideoWriter('/home/pi/Desktop/cone/video.avi', cv2.cv.CV_FOURCC('M', 'J', 'P', 'G'), 1, (640, 240)) for name in images: tmp_list = name.split('/') title = tmp_list[-1] image_real = cv2.imread("/home/pi/Desktop/cone/real/" + title) image_filtered = cv2.imread("/home/pi/Desktop/cone/filtered/" + title) vis = np.concatenate((image_real, image_filtered), axis=1) time.sleep(0.1) cv2.imshow("vis", vis) key = cv2.waitKey(1) & 0xFF video.write(vis)
def main_worker(rank, cfg): # Initialize the worker distributed = init_worker(rank, cfg) # Initialize the random seed if cfg.seed is not None: torch.manual_seed(cfg.seed) # Initialize the PyTorch device device_id = cfg.device_id + rank device = init_device(cfg, id=device_id) # Initialize the model model = get_model(cfg) model.to(device) if distributed: model = nn.parallel.DistributedDataParallel(model, device_ids=[device_id]) # Initialize the loss function criterion = get_loss_function(cfg) criterion.to(device) # Initialize the optimizer optimizer = optim.Adam(model.parameters(), lr=1) # Check whether the result already exists result_dir = get_result_dir(cfg) resume = os.path.isdir(result_dir) # Sync the workers (required due to the previous isdir check) if distributed: dist.barrier() # Start or resume training if resume: if rank == 0: print('Resuming result:', cfg.result) # Load and verify the config result_cfg = load_config(result_dir) if set(result_cfg.features) != set(cfg.features): error('input feature set mismatch') # Restore the latest checkpoint last_epoch = get_latest_checkpoint_epoch(result_dir) checkpoint = load_checkpoint(result_dir, device, last_epoch, model, optimizer) step = checkpoint['step'] else: if rank == 0: print('Result:', cfg.result) os.makedirs(result_dir) # Save the config save_config(result_dir, cfg) # Save the source code src_filenames = glob(os.path.join(os.path.dirname(sys.argv[0]), '*.py')) src_zip_filename = os.path.join(result_dir, 'src.zip') save_zip(src_zip_filename, src_filenames) last_epoch = 0 step = 0 # Make sure all workers have loaded the checkpoint if distributed: dist.barrier() start_epoch = last_epoch + 1 if start_epoch > cfg.num_epochs: exit() # nothing to do # Reset the random seed if resuming result if cfg.seed is not None and start_epoch > 1: seed = cfg.seed + start_epoch - 1 torch.manual_seed(seed) # Initialize the training dataset train_data = TrainingDataset(cfg, cfg.train_data) if len(train_data) > 0: if rank == 0: print('Training images:', train_data.num_images) else: error('no training images') train_loader, train_sampler = get_data_loader(rank, cfg, train_data, shuffle=True) train_steps_per_epoch = len(train_loader) # Initialize the validation dataset valid_data = ValidationDataset(cfg, cfg.valid_data) if len(valid_data) > 0: if rank == 0: print('Validation images:', valid_data.num_images) valid_loader, valid_sampler = get_data_loader(rank, cfg, valid_data, shuffle=False) valid_steps_per_epoch = len(valid_loader) # Initialize the learning rate scheduler lr_scheduler = optim.lr_scheduler.OneCycleLR( optimizer, max_lr=cfg.max_lr, total_steps=cfg.num_epochs, pct_start=cfg.lr_warmup, anneal_strategy='cos', div_factor=(25. if cfg.lr is None else cfg.max_lr / cfg.lr), final_div_factor=1e4, last_epoch=last_epoch-1) if lr_scheduler.last_epoch != last_epoch: error('failed to restore LR scheduler state') # Check whether AMP is enabled amp_enabled = cfg.precision == 'mixed' if amp_enabled: # Initialize the gradient scaler scaler = amp.GradScaler() # Initialize the summary writer log_dir = get_result_log_dir(result_dir) if rank == 0: summary_writer = SummaryWriter(log_dir) if step == 0: summary_writer.add_scalar('learning_rate', lr_scheduler.get_last_lr()[0], 0) # Training and evaluation loops if rank == 0: print() progress_format = '%-5s %' + str(len(str(cfg.num_epochs))) + 'd/%d:' % cfg.num_epochs total_start_time = time.time() for epoch in range(start_epoch, cfg.num_epochs+1): if rank == 0: start_time = time.time() progress = ProgressBar(train_steps_per_epoch, progress_format % ('Train', epoch)) # Switch to training mode model.train() train_loss = 0. # Iterate over the batches if distributed: train_sampler.set_epoch(epoch) for i, batch in enumerate(train_loader, 0): # Get the batch input, target = batch input = input.to(device, non_blocking=True) target = target.to(device, non_blocking=True) if not amp_enabled: input = input.float() target = target.float() # Run a training step optimizer.zero_grad() with amp.autocast(enabled=amp_enabled): output = model(input) loss = criterion(output, target) if amp_enabled: scaler.scale(loss).backward() scaler.step(optimizer) scaler.update() else: loss.backward() optimizer.step() # Next step step += 1 train_loss += loss if rank == 0: progress.next() # Get and update the learning rate lr = lr_scheduler.get_last_lr()[0] lr_scheduler.step() # Compute the average training loss if distributed: dist.all_reduce(train_loss, op=dist.ReduceOp.SUM) train_loss = train_loss.item() / (train_steps_per_epoch * cfg.num_devices) # Write summary if rank == 0: summary_writer.add_scalar('learning_rate', lr, epoch) summary_writer.add_scalar('loss', train_loss, epoch) # Print stats if rank == 0: duration = time.time() - start_time total_duration = time.time() - total_start_time images_per_sec = len(train_data) / duration eta = ((cfg.num_epochs - epoch) * total_duration / (epoch + 1 - start_epoch)) progress.finish('loss=%.6f, lr=%.6f (%.1f images/s, %s, eta %s)' % (train_loss, lr, images_per_sec, format_time(duration), format_time(eta, precision=2))) if ((cfg.num_valid_epochs > 0 and epoch % cfg.num_valid_epochs == 0) or epoch == cfg.num_epochs) \ and len(valid_data) > 0: # Validation if rank == 0: start_time = time.time() progress = ProgressBar(valid_steps_per_epoch, progress_format % ('Valid', epoch)) # Switch to evaluation mode model.eval() valid_loss = 0. # Iterate over the batches with torch.no_grad(): for _, batch in enumerate(valid_loader, 0): # Get the batch input, target = batch input = input.to(device, non_blocking=True).float() target = target.to(device, non_blocking=True).float() # Run a validation step loss = criterion(model(input), target) # Next step valid_loss += loss if rank == 0: progress.next() # Compute the average validation loss if distributed: dist.all_reduce(valid_loss, op=dist.ReduceOp.SUM) valid_loss = valid_loss.item() / (valid_steps_per_epoch * cfg.num_devices) # Write summary if rank == 0: summary_writer.add_scalar('valid_loss', valid_loss, epoch) # Print stats if rank == 0: duration = time.time() - start_time images_per_sec = len(valid_data) / duration progress.finish('valid_loss=%.6f (%.1f images/s, %.1fs)' % (valid_loss, images_per_sec, duration)) if (rank == 0) and ((cfg.num_save_epochs > 0 and epoch % cfg.num_save_epochs == 0) or epoch == cfg.num_epochs): # Save a checkpoint save_checkpoint(result_dir, epoch, step, model, optimizer) # Print final stats if rank == 0: total_duration = time.time() - total_start_time print('\nFinished (%s)' % format_time(total_duration)) # Cleanup cleanup_worker(cfg)
def delete_products(current_products, products_to_update): """ Удаление продуктов, которых нет в новом xml или которые удалили из директории с дополнительными изображениями. Удаление индексов продуктов из модели с faiss индексами :param current_products: датафрейм из нового xml-файла :param products_to_update: текущий датафрейм :return: датафрейм products_to_update с удаленными продуктами """ logging.info('Удаление продуктов') idx_to_remove = [] # удаление продуктов, которых нет в новом xml # смотрим, какие продукты нужно удалить, исходя из отсутствия артикулов продуктов текущей модели в новом xml products_to_update_vendor_code = set(products_to_update['vendor_code']) current_products_vendor_code = set(current_products['vendor_code']) products_to_update_vendor_code.difference_update( current_products_vendor_code) if products_to_update_vendor_code: for index, row in products_to_update.iterrows(): if row['vendor_code'] in products_to_update_vendor_code: idx_to_remove.append(index) products_to_update.drop(index, inplace=True) # удаление строк с изображениями, которых уже нет в директории current_product_files = set( glob.glob(config.PATH_TO_PRODUCT_FOLDER + '/*/*/*')) product_files_to_update = set( products_to_update[~products_to_update['picture'].str. startswith('http')]['picture'].values) product_files_to_update.difference_update(current_product_files) if product_files_to_update: for file in product_files_to_update: index = products_to_update[products_to_update['picture'] == file].index[0] products_to_update.drop(index, inplace=True) idx_to_remove.append(index) products_to_update.reset_index(inplace=True, drop=True) # если ничего не удаляли, возвращаем неизмененный датафрейм if not idx_to_remove: logging.info('Продуктов для удаления нет') return products_to_update # удаление из faiss соответствующих векторов logging.info(f'Удаление индексов из faiss [{len(idx_to_remove)} строк]') index = faiss.read_index(config.PATH_TO_FAISS_INDEX) vectors = [index.reconstruct(i) for i in range(index.ntotal)] vectors_without_removed = [ vectors[i] for i in range(len(vectors)) if i not in idx_to_remove ] updated_index = faiss.IndexFlatL2(2048) updated_index = faiss.IndexIDMap2(updated_index) updated_index.add_with_ids( np.vstack(vectors_without_removed), np.hstack([i for i in range(len(vectors_without_removed))])) logging.info('Запись') faiss.write_index(updated_index, config.PATH_TO_FAISS_INDEX) products_to_update.to_pickle(config.PATH_TO_PRODUCT_DATASET) assert updated_index.ntotal == products_to_update.shape[0] logging.info(f'Удалено {len(idx_to_remove)} строк') return products_to_update
def add_additional_images(products): """ Добавление в датафрейм изображений продуктов в директории config.PATH_TO_PRODUCT_FOLDER :param products: датафрейм с текущими продуктами :return: датафрейм products с добавленными строками с дополнительными изображениями продуктов, список idx_to_add с индексами добавленных строк для их дальнейшей обработки """ logging.info( f'Добавление в датафрейм изображений из директории {config.PATH_TO_PRODUCT_FOLDER}' ) product_files = glob.glob(config.PATH_TO_PRODUCT_FOLDER + '/*/*') # счетчики для отслеживания количества обработанных продуктов count_all_products = 0 count_absent_products = 0 count_all_images = 0 count_error_images = 0 # последний индекс в датафрейме, после которого можно добавлять новые строки curr_index = products.index.max() # индексы добавленных строк idx_to_add = [] # читаем отдельно каждую папку с отдельным продуктом for product_path in product_files: count_all_products += 1 vendor_code = product_path.split('/')[3] # проверяем, если в датасете продукт с таким артикулом product_to_duplicate = products.loc[products['vendor_code'] == str( vendor_code)] if product_to_duplicate.empty: count_absent_products += 1 logging.warning(f'В базе нет продукта с артикулом {vendor_code}') continue # читаем все изображения текущего продукта pictures = glob.glob(product_path + '/*') for picture_full_path in pictures: picture_name = picture_full_path.replace( config.PATH_TO_PRODUCT_FOLDER, '') # если такое изображение уже есть в датафрейме, читаем следующее if picture_name in products['picture'].values: continue # открываем изображение в зависимости от его расширения count_all_images += 1 try: utils.open_local_image(picture_name) # дублируем строку с найденным продуктом в датасете по текущему артикулу # меняем только путь к изображению curr_index += 1 products.loc[curr_index] = product_to_duplicate.values[0] products.loc[curr_index]['picture'] = picture_name idx_to_add.append(curr_index) # если изображение не открывается, пишем в лог except Exception as ex: count_error_images += 1 logging.warning( f'Не получилось открыть изображение, ошибка: {ex}') if count_all_products % 500 == 0: logging.debug(f'Обработано строк: {count_all_products}') logging.info( f'Из {count_all_products} продуктов в базе не нашлось {count_absent_products}' ) logging.info( f'Из {count_all_images} изображений продуктов, которые нашлись в базе, не обработано {count_error_images}' ) return products, idx_to_add
"Topic :: Multimedia :: Graphics :: Viewers", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', ], cmdclass={"build_ext": pil_build_ext}, ext_modules=[Extension("PIL._imaging", ["_imaging.c"])], include_package_data=True, packages=find_packages(), scripts=glob.glob("Scripts/*.py"), install_requires=['olefile'], test_suite='nose.collector', keywords=["Imaging", ], license='Standard PIL License', zip_safe=not debug_build(), ) except RequiredDependencyException as err: msg = """ The headers or library files could not be found for %s, a required dependency when compiling Pillow from source. Please see the install instructions at: https://pillow.readthedocs.io/en/latest/installation.html """ % (str(err))
counter = 0 #noiseFunctions = funcGenerator() source_paths = [] # Array of file paths to be reviewed shape_vals = [] # Array containing the shape feature values of the candidates skew_vals = [] # Array containing the skewness feature values of the candidates kurt_vals = [] # Array containing the kurtosis feature values of the candidates kstest_vals = [] # Array containing the ks-test feature values of the candidates class_vals = [] # Array containing the classification labels of the candidates timer1 = [] timer2 = [] # Loops through all .dat files to store them in the 'source_paths' array for file in glob.glob(os.getcwd() + '\idir\\' + "*.dat"): source_paths.append(file) # Import dedispersion plan df_ddp = pd.read_csv("dd_plan.txt") # Setup array for step limits dd_DM = np.array(df_ddp["DM_stop"]) dd_step = np.array(df_ddp["DM_step"]) # Constructing DM_poss array of possible DM values from ddp DM_start = df_ddp["DM_start"] DM_stop = df_ddp["DM_stop"] DM_step = df_ddp["DM_step"] DM_poss = [0.0]
] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False # -- Options for autosummary ---------------------------------------------- autosummary_generate = glob.glob('*.rst') # -- Options for numpydoc ------------------------------------------------- # Generate plots for example sections numpydoc_use_plots = True # If we don't turn numpydoc's toctree generation off, Sphinx will warn about # the toctree referencing missing document(s). This appears to be related to # generating docs for classes with a __call__ method. numpydoc_class_members_toctree = False #------------------------------------------------------------------------------ # Plot #------------------------------------------------------------------------------ plot_pre_code = """ import numpy as np import scipy as sp
#Diccionario con info de plot: se lee la info de todos los parametrizaciones ListPlotVar = al.get_modelConfig_lines(ListConfig, '-p', Calib_Storage='Plot',PlotType='Humedad_map') DictStore = al.get_modelConfig_lines(ListConfig, '-s', 'Store') #construye las listas para plotear en paralelo ListaEjec = [] for l in ListPlotVar: #Se define ruta donde se leeran los resultados a plotear ruta_in = ruta_sto + DictStore['-s '+l]['Nombre'] #Se crea un folder en el que se van a contener las imagenes de cada parametrizacion asignada #Mira la ruta del folder y si no existe la crea ruta_folder = ruta_Hsim+l+'/' Esta = glob.glob(ruta_folder) if len(Esta) == 0: os.system('mkdir '+ruta_folder) #Obtiene las rutas de los archivos de salida ruta_out_png = ruta_folder +'Humedad'+l+'_'+args.date+'.png' ruta_out_txt = ruta_folder +'Humedad'+l+'_'+args.date+'.txt' #Lee los binarios de humedad para la cuenca de cada parametrizacion v,r = wmf.models.read_float_basin_ncol(ruta_in,args.record, cu.ncells, 5) #Se organiza la lista con parametros necesarios para plotear los mapas con la funcion que sigue ListaEjec.append([ruta_in, ruta_out_png, ruta_out_txt, v, l]) #------------------------------------------------------------------------------------------------------- #Se generan los plots de humedad de cada parametrizacion #------------------------------------------------------------------------------------------------------- def Plot_Hsim(Lista):
import numpy as np # Read test data from file test_data = pd.read_excel('../resources/pozyxAPI_only_localization_dane_testowe_i_dystrybuanta.xlsx') test_mes_x = test_data.pop('measurement x') test_mes_y = test_data.pop('measurement y') test_tar_x = test_data.pop('reference x') test_tar_y = test_data.pop('reference y') test_mes = pd.concat([test_mes_x, test_mes_y], axis=1) test_tar = pd.concat([test_tar_x, test_tar_y], axis=1) # Read training & target data from files path = '../resources/' column_names = ['0/timestamp', 't', 'no', 'measurement x', 'measurement y', 'reference x', 'reference y'] all_files = glob.glob(os.path.join(path, 'pozyxAPI_only_localization_measurement*.xlsx')) df_from_each_file = (pd.read_excel(f, names=column_names) for f in all_files) concatenated_df = pd.concat(df_from_each_file, ignore_index=True) mes_x = concatenated_df.pop('measurement x') mes_y = concatenated_df.pop('measurement y') training_data = pd.concat([mes_x, mes_y], axis=1) tar_x = concatenated_df.pop('reference x') tar_y = concatenated_df.pop('reference y') target_data = pd.concat([tar_x, tar_y], axis=1) # Add offset to data for better readability training_data = (training_data.astype('float32') + 2000) / 10000 target_data = (target_data.astype('float32') + 2000) / 10000
def install_requirements_conda(ctx: Context): """Install all requirements, including tools used by Roberto.""" # Collect all parameters determining the install commands (to good # approximation) and turn them into a hash. # Some conda requirements are included by default because they must be present: # - conda: to make sure it is always up to date. # - conda-build: to have conda-render for getting requirements from recipes. conda_reqs = set(["conda"]) pip_reqs = set([]) recipe_dirs = [] # Add project as a tool because it also contains requirements. tools = [ctx.project] for package in ctx.project.packages: for toolname in package.tools: tools.append(ctx.tools[toolname]) recipe_dir = os.path.join(package.path, "tools", "conda.recipe") if os.path.isdir(recipe_dir): recipe_dirs.append(recipe_dir) else: print("Skipping recipe {}. (directory does not exist)".format(recipe_dir)) for tool in tools: for conda_req, pip_req in tool.get("requirements", []): if conda_req is None: pip_reqs.add(pip_req) conda_reqs.add("pip") else: conda_reqs.add(conda_req) req_hash = compute_req_hash( set("conda:" + conda_req for conda_req in conda_reqs) | set("pip:" + pip_req for pip_req in pip_reqs), sum([glob(os.path.join(recipe_dir, "*")) for recipe_dir in recipe_dirs], []) ) fn_skip = os.path.join(ctx.testenv.path, ".skip_install") if check_install_requirements(fn_skip, req_hash): with ctx.prefix(ctx.conda.activate_base): # Update conda packages in the base env. Conda packages in the dev env # tend to be ignored. ctx.run("conda install --update-deps -y {}".format( " ".join("'{}'".format(conda_req) for conda_req in conda_reqs if conda_req.startswith('conda')))) with ctx.prefix(ctx.testenv.activate): # Update packages already installed ctx.run("conda update --all -y") # Update and install other requirements for Roberto, in the dev env. ctx.run("conda install --update-deps -y {}".format(" ".join( "'{}'".format(conda_req) for conda_req in conda_reqs if not conda_req.startswith('conda')))) print("Rendering conda package, extracting requirements, which will be installed.") # Install dependencies from recipes, excluding own packages. own_conda_reqs = [package.dist_name for package in ctx.project.packages] for recipe_dir in recipe_dirs: # Send the output of conda render to a temporary directory. with tempfile.TemporaryDirectory() as tmpdir: rendered_path = os.path.join(tmpdir, "rendered.yml") ctx.run( "conda render -f {} {} --variants {}".format( rendered_path, recipe_dir, ctx.conda.variants)) with open(rendered_path) as f: rendered = yaml.safe_load(f) # Build a (simplified) list of requirements and install. dep_conda_reqs = set([]) req_sources = [ ("requirements", 'build'), ("requirements", 'host'), ("requirements", 'run'), ("test", 'requires'), ] for req_section, req_type in req_sources: for recipe_req in rendered.get(req_section, {}).get(req_type, []): words = recipe_req.split() if words[0] not in own_conda_reqs: dep_conda_reqs.add(" ".join(words[:2])) ctx.run("conda install --update-deps -y {}".format(" ".join( "'{}'".format(conda_req) for conda_req in dep_conda_reqs))) # Update and install requirements for Roberto from pip, if any. if pip_reqs: ctx.run("pip install --upgrade {}".format(" ".join( "'{}'".format(pip_req) for pip_req in pip_reqs))) # Update the timestamp on the skip file. with open(fn_skip, 'w') as f: f.write(req_hash + '\n')
datasetfn1 = args.datasetfn1 datasetfn2 = args.datasetfn2 import utilities as util import models as models import data_generation as data_generation import plotting as plotting use_dir = False if configfile: cfg_fns = [configfile] elif configfile_dir: if configfile_dir[-1] != '/': configfile_dir += '/' cfg_fns = glob.glob(configfile_dir + '*') cfg_fns.sort() if endidx: assert startidx <= endidx, "Startidx is greater than endidx...not judging, just letting you know..." cfg_fns = cfg_fns[startidx:endidx + 1] elif use_dir: # cfg_fns = "config_files/noise_model.ini" cfg_fns = glob.glob('exp_bm_gaussian_11am/*') cfg_fns.sort() if endidx: assert startidx <= endidx, "Startidx is greater than endidx...not judging, just letting you know..." cfg_fns = cfg_fns[startidx:endidx] #cfg_fns = glob.glob('test_batch/*') else: #cfg_fns = ["config_files/noise_baseModel.ini"] #cfg_fns = ["config_files/broken_structured.ini"]
('.'.join((str(v) for v in version_required)), '.'.join((str(v) for v in version_running )),), file=sys.stderr) sys.exit(1) def resize_mask(filename, width, height, output_folder=""): if not os.path.exists(filename): # remove file extension and try again ext = filename.rsplit(".",1) if len(ext)>1: filename = ext[0] filename = filename + ".npy" tmp = np.load(filename) name = os.path.basename(filename).replace(".npy","") tmp = cv2.resize(tmp.reshape(tmp.shape[1], tmp.shape[2]), (width, height), interpolation = cv2.INTER_CUBIC) _, tmp = cv2.threshold(tmp,0.5,255,cv2.THRESH_BINARY) if output_folder: # print(os.path.join(output_folder, name + "_segmentation.png")) cv2.imwrite(os.path.join(output_folder, name + "_segmentation.png"), tmp) return tmp size = int(sys.argv[1]) source = sys.argv[2] dest = sys.argv[3] print("Resizing images to %dx%d from %s to %s" % (size, size, source, dest)) width, height = size, size output_folder = dest for fname in tqdm(glob.glob(os.path.join(source, "*"))): resized_img = resize_mask(fname, width, height, output_folder=output_folder) assert resized_img.shape==(width, height)
def __init__(self, input_dim=(3, 32, 32), num_filters=None, filter_size=5, hidden_dims=None, num_classes=10, weight_scale=1e-3, reg=0.1, dtype=np.float32, use_batchnorm=False, dropout=0, seed=None, loadData=None, predict_fn=None, augment_fn=None): """ predic_fn, augment_fn: for data augmentation """ self.params = {} self.reg = reg self.dtype = dtype self.num_filters = len(num_filters) self.filter_size = filter_size self.bn_batchnorm = use_batchnorm self.use_dropout = (dropout > 0) ############# the total number of layers including conv layer and affine layer####### self.num_layers = self.num_filters + len(hidden_dims) + 1 ##################################################################################### # print "how many layers ? ",self.num_layers self.bn_params = [] self.dropout_param = {} self.predict_fn = predict_fn self.augment_fn = augment_fn if augment_fn is not None: input_dim = (3, 28, 28) self.input_dim = input_dim if loadData is not None: print "Load Data is ", loadData for f in glob.glob("%s/convNet_params_*.npy" % loadData): name_lst = op.splitext(op.basename(f))[0].split("_") if len(name_lst) == 3: param = name_lst[2] if param == "dropout": self.dropout_param = self.load_param(f) elif param == "bn": self.bn_params = self.load_param(f) else: self.params[param] = self.load_param( f) # W_i,b_i,beta_i, gamma_i print self.params[param].shape, print "load parameter %s successfully" % param return C, H, W = input_dim assert filter_size % 2 == 1, 'Filter size must be odd: got %d' % filter_size all_filters = np.array([C]) all_filters = np.concatenate((all_filters, np.array(num_filters)), axis=0) for i in range(self.num_filters): t = i + 1 self.params['W%d' % t] = weight_scale * np.random.randn( all_filters[t], all_filters[t - 1], filter_size, filter_size) self.params['b%d' % t] = np.zeros(all_filters[t]) if self.bn_batchnorm is True: self.params['gamma%d' % t] = np.random.randn(all_filters[t]) self.params['beta%d' % t] = np.random.randn(all_filters[t]) all_hidden_layers = np.array( [H * W * all_filters[-1] / np.power(4, self.num_filters)]) a = np.array(hidden_dims) all_hidden_layers = np.concatenate((all_hidden_layers, a), axis=0) b = np.array([num_classes]) all_hidden_layers = np.concatenate((all_hidden_layers, b)) length = len(all_hidden_layers) - 1 for i in range(length): t = i + self.num_filters + 1 self.params['W%d' % t] = weight_scale * np.random.randn( all_hidden_layers[i], all_hidden_layers[i + 1]) self.params['b%d' % t] = np.zeros(all_hidden_layers[i + 1]) if self.bn_batchnorm is True and i < length - 1: self.params['gamma%d' % t] = np.random.randn( all_hidden_layers[i + 1]) self.params['beta%d' % t] = np.random.randn( all_hidden_layers[i + 1]) if self.bn_batchnorm: self.bn_params = [{ 'mode': 'train' } for i in xrange(self.num_layers)] if self.use_dropout is True: self.dropout_param = {'mode': 'train', 'p': dropout} if seed is not None: self.dropout_param['seed'] = seed for k, v in self.params.iteritems(): self.params[k] = v.astype(dtype)
self.hdu_list = fitsio.FITS(filepath) #, iter_row_buffer=25) @property def dataHDU(self): return self.hdu_list[1] @property def column_names(self): if self._column_names == None: row1 = self.dataHDU.read(rows=[0,0]) self._column_names = [x.lower() for x in row1.dtype.names] return self._column_names #column_names = None for filepath in glob.glob(os.path.join(base_directory, "8162", "photoRun-*.fits")): # The file contains a list of Field records. session.begin() photoRunFile = PhotoRunFile(filepath) filename = os.path.basename(filepath) if args.verbose: print("Processing {0}...".format(filename)) try: session.query(Run).filter(Run.filename == filename).one() #print ("found") continue # already in database, go to next file
def tearDown(self): files = glob.glob('test_lc0*') print files for name in files: os.remove(name)
def evaluation_with_statistics(model_dir, statistics_file , silver_all_seed_json, silver_sample_seed_json, to_file): statistics_list = read_dict_from_csv(statistics_file) dict_result = {} positive_count = 0 negative_count = 0 NULL_count = 0 for x in statistics_list: P, positive, negative, null_ = x["P"], x["positive"], x["negative"], x["NULL"] P = unicode(P) dict_result[P] = {} dict_result[P]["positive"] = int(positive) dict_result[P]["negative"] = int(negative) dict_result[P]["NULL"] = int(null_) positive_count += int(positive) negative_count += int(negative) NULL_count += int(null_) dict_all_seed = json.load(open(silver_all_seed_json)) dict_sample_seed = json.load(open(silver_sample_seed_json)) # 分子, 也就是对的个数 precision_molecular = 0 # 分母, 也就是预测的个数 precision_denominator = 0 recall_molecular = 0 recall_denominator = 0 # N = 0 macro_precision = 0.0 macro_recall = 0.0 for predict_file in glob.glob('%s/*/predict.json' % model_dir): dict_predict = json.load(open(predict_file)) for P in dict_predict: predicts = set([(s, o) for s, o, prob in dict_predict[P]]) silver_all = set(map(tuple, dict_all_seed[P])) silver_sample = set(map(tuple, dict_sample_seed[P])) precision_molecular += len(predicts & silver_all) precision_denominator += len(predicts) recall_molecular += len(predicts & silver_sample) recall_denominator += len(silver_sample) # 每个P也计算一下吧 if len(predicts) == 0: precision = 0 recall = 0 else: precision = len(predicts & silver_all) / float(len(predicts)) * 100 recall = len(predicts & silver_sample) / float(len(silver_sample)) * 100 N += 1 macro_precision += precision macro_recall += recall dict_result[P]["precision"] = "%d / %d = %.2f%%" %\ (len(predicts & silver_all), len(predicts), precision) dict_result[P]["recall"] = "%d / %d = %.2f%%" % (len(predicts & silver_sample), len(silver_sample), recall) s = "%s\tprecision: %d / %d = %.2f%%\trecall: %d / %d = %.2f%%" % \ ( P, len(predicts & silver_all), len(predicts), precision, len(predicts & silver_sample), len(silver_sample), recall) print s micro_precision = precision_molecular / float(precision_denominator) * 100 micro_recall = recall_molecular / float(recall_denominator) * 100 macro_precision = macro_precision / N macro_recall = macro_recall / N dict_result["Micro"] = {} dict_result["Micro"]["positive"] = positive_count dict_result["Micro"]["negative"] = negative_count dict_result["Micro"]["NULL"] = NULL_count dict_result["Micro"]["precision"] = "%d / %d = %.2f%%" % (precision_molecular, precision_denominator, micro_precision) dict_result["Micro"]["recall"] = "%d / %d = %.2f%%" % (recall_molecular, recall_denominator, micro_recall) dict_result["Macro"] = {} dict_result["Macro"]["positive"] = "" dict_result["Macro"]["negative"] = "" dict_result["Macro"]["NULL"] = "" dict_result["Macro"]["precision"] = "%.2f%%" % (macro_precision) dict_result["Macro"]["recall"] = "%.2f%%" % (macro_recall) s = "%s\tprecision: %d / %d = %.2f%%\trecall: %d / %d = %.2f%%" % \ ( "Micro", precision_molecular, precision_denominator, micro_precision, recall_molecular, recall_denominator, micro_recall) print s fout = codecs.open(to_file, "w", encoding="utf-8") fout.write("P,positive,negative,NULL,precision,recall\n") for P in sorted(dict_result.keys()): fout.write("%s,%s,%s,%s,%s,%s\n" % (P, dict_result[P]["positive"], dict_result[P]["negative"], dict_result[P]["NULL"], dict_result[P]["precision"], dict_result[P]["recall"], ) ) fout.close()
# "櫻井愛菜", # "田中沙季", # "増崎セリナ", # "小野晴茄", # "澤田守杏", # "播磨ここね", # "田中希湖", # "石田美咲希", # ] input_file = 'output/measure/' output_file = 'compare/' # %% file_paths = glob(input_file+'*.xlsx') file_paths # %% def extract(file_path): df = pd.read_excel(file_path) column = df.iloc[0, 2] df = df.iloc[:, [1, 12]] df = df.rename(columns={'20本ジャンプ (秒)':column}) return df # %% df = pd.DataFrame()
import glob import os import shutil import sys from typing import Any, Dict, List from argoverse.utils.json_utils import read_json_file, save_json_dict root_dir = sys.argv[1] print("root dir = ", root_dir) print("updating track_labels_amodal folders...") list_log_folders = glob.glob(os.path.join(root_dir, "*")) if len(list_log_folders) == 0: print("Not file founded.") else: for ind_log, path_log in enumerate(list_log_folders): print("Processing %d/%d" % (ind_log + 1, len(list_log_folders))) list_path_label_persweep = glob.glob( os.path.join(path_log, "per_sweep_annotations_amodal", "*")) list_path_label_persweep.sort() dist_track_labels: Dict[str, List[Any]] = {} for path_label_persweep in list_path_label_persweep: data = read_json_file(path_label_persweep) for data_obj in data: id_obj = data_obj["track_label_uuid"] if id_obj not in dist_track_labels.keys(): dist_track_labels[id_obj] = [] dist_track_labels[id_obj].append(data_obj)
# DemoOS.py from os.path import * #print(dir(os.path)) print(abspath("python.exe")) print(basename("c:\\python\\python.exe")) print(exists("c:\\python\\python.exe")) print(getsize("c:\\python\\python.exe")) #운영체제에 있는 명령어를 실행(cmd --> cd \, cd work2) from os import * # c:\work2 --> c: --> c:\work2 print("현재 작업 폴더:", getcwd()) chdir("..") chdir("c:\\work2") print("현재 폴더:", getcwd()) #외부의 실행파일 수행 #system("notepad.exe") #파일, 폴더 리스트 import glob print(glob.glob("*.py")) print("="*20) for item in glob.glob("*.*"): print(item)
'lb-tp-approx2-weak-scaling' ] } if __name__ == '__main__': args = parser.parse_args() failed_plots = [] not_ready_plots = [] for plot, requirements in plot_scripts.items(): isReady = True for case in args.testcases.split(','): for req in requirements: directory = os.path.join(args.indir, case, req) if os.path.isdir(directory) and len( glob.glob(directory + '/.extracted')) > 0: continue else: isReady = False break if isReady: cmd = [ 'python', os.path.join(this_directory, plot), '-i', args.indir, '-c', args.testcases ] output = subprocess.run(cmd, stdout=sys.stdout, stderr=subprocess.STDOUT, env=os.environ.copy()) if output.returncode != 0:
def documents(name, **kwargs): directory = kwargs['directory'] for doc_path in glob.glob(directory+'*.md'): yield document_from_path(doc_path)
2) Can we find any evidence against hull hypothesis that neural activations patterns on the cortex does remain the same across sessions ? * In other words, there is no evidence of cortical reorganization. """ #%%############################################################################## # Read the EEG data from uhClass import MRCP mainDir = '/home/hero/uhdata/freesurfer' subject = 'S9023' os.chdir(os.path.join(mainDir, subject)) eegfiles = glob.glob('*-epo.fif') #%% eeg = [] for eegfilename in eegfiles: if not ('_ses1_' in eegfilename or '_ses2_' in eegfilename): # eegfilename = 'S9017_ses3_closeloop_block0000-epo.fif' ep = MRCP.readEEGepoch(eegfilename, mainDir) ep = ep.interpolate_bads(reset_bads='True', mode='accurate') ep.info['bads'] = [] ep.resample(100) eeg.append(ep) #%% Plot EEG #from uhClass import MRCP #filename = eegfilename.split('_s')[0]
def blosc_extension(): info('setting up Blosc extension') # setup blosc extension blosc_sources = [] extra_compile_args = [] include_dirs = [] define_macros = [] # generic setup blosc_sources += [ f for f in glob('c-blosc/blosc/*.c') if 'avx2' not in f and 'sse2' not in f ] blosc_sources += glob('c-blosc/internal-complibs/lz4*/*.c') blosc_sources += glob('c-blosc/internal-complibs/snappy*/*.cc') blosc_sources += glob('c-blosc/internal-complibs/zlib*/*.c') blosc_sources += glob('c-blosc/internal-complibs/zstd*/common/*.c') blosc_sources += glob('c-blosc/internal-complibs/zstd*/compress/*.c') blosc_sources += glob('c-blosc/internal-complibs/zstd*/decompress/*.c') blosc_sources += glob('c-blosc/internal-complibs/zstd*/dictBuilder/*.c') include_dirs += [os.path.join('c-blosc', 'blosc')] include_dirs += [ d for d in glob('c-blosc/internal-complibs/*') if os.path.isdir(d) ] include_dirs += [ d for d in glob('c-blosc/internal-complibs/*/*') if os.path.isdir(d) ] define_macros += [('HAVE_LZ4', 1), ('HAVE_SNAPPY', 1), ('HAVE_ZLIB', 1), ('HAVE_ZSTD', 1)] # define_macros += [('CYTHON_TRACE', '1')] # determine CPU support for SSE2 and AVX2 cpu_info = cpuinfo.get_cpu_info() # SSE2 if 'sse2' in cpu_info['flags']: info('SSE2 detected') extra_compile_args.append('-DSHUFFLE_SSE2_ENABLED') blosc_sources += [f for f in glob('c-blosc/blosc/*.c') if 'sse2' in f] if os.name == 'posix': extra_compile_args.append('-msse2') elif os.name == 'nt': define_macros += [('__SSE2__', 1)] # AVX2 if 'avx2' in cpu_info['flags']: info('AVX2 detected') extra_compile_args.append('-DSHUFFLE_AVX2_ENABLED') blosc_sources += [f for f in glob('c-blosc/blosc/*.c') if 'avx2' in f] if os.name == 'posix': extra_compile_args.append('-mavx2') elif os.name == 'nt': define_macros += [('__AVX2__', 1)] # workaround lack of support for "inline" in MSVC when building for Python # 2.7 64-bit if PY2 and os.name == 'nt': extra_compile_args.append('-Dinline=__inline') if have_cython: sources = ['numcodecs/blosc.pyx'] else: sources = ['numcodecs/blosc.c'] # define extension module extensions = [ Extension( 'numcodecs.blosc', sources=sources + blosc_sources, include_dirs=include_dirs, define_macros=define_macros, extra_compile_args=extra_compile_args, ), ] if have_cython: extensions = cythonize(extensions) return extensions
# ============================================================================== # -- find carla module --------------------------------------------------------- # ============================================================================== import glob import os import sys import pandas as pd import csv try: sys.path.append(glob.glob('C:/Coursera/carla/PythonAPI/carla/dist/carla-*%d.%d-%s.egg' % ( sys.version_info.major, sys.version_info.minor, 'win-amd64' if os.name == 'nt' else 'linux-x86_64'))[0]) except IndexError: pass # ============================================================================== # -- imports ------------------------------------------------------------------- # ============================================================================== import carla from carla import ColorConverter as cc import argparse
def __init__(self): self.base_dir = '/sys/bus/w1/devices/' self.device_folder = glob.glob(self.base_dir + '28*')[0] self.device_file = self.device_folder + '/w1_slave'
import pandas as pd import numpy as np import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import seaborn as sns from mpl_toolkits.axes_grid1.inset_locator import inset_axes import sqlite3 import glob sns.set(font_scale=1.5, style="white") files = glob.glob('*_traj_merged.db') dfs = [] for fi in files: s = fi.split("_") opt = s[3] if opt == '1': mu = float(s[5]) print(fi,mu) c = sqlite3.connect(fi) df = pd.read_sql('select repid,freq,origin,esize from freqs where repid < 1 and generation == 50000',c) # group by repid,pos,origin',c) df.drop_duplicates(inplace=True) df['mu']=[mu]*len(df.index) dfs.append(df) c.close() df = pd.concat(dfs) gr = df.groupby(['mu'])
#!/usr/bin/env python #coding:utf-8 import os, sys, re, csv, glob wordlist = list() for fname in glob.glob('../../../forth/primitives/*.asm'): wordlist = wordlist + re.findall('\n\s+[;/]+\s+word:\s+(\S+)', open(fname).read()) def texify(x): for word in x: for char in "\\$_&{}%#": word = word.replace(char, '\\' + char) yield word.replace('\\\\', '\\symbol{92}') print '\n'.join(r'\fw{%s}' % x for x in texify(sorted(wordlist)))
args = VarParsing.VarParsing('analysis') args.register('inputFile', '', args.multiplicity.list, args.varType.string, "Input file or template for glob") args.outputFile = '' args.parseArguments() ''' ##################### Input ################### ''' process.maxEvents = cms.untracked.PSet( input=cms.untracked.int32(args.maxEvents)) from glob import glob if args.inputFile: if len(args.inputFile) == 1 and '*' in args.inputFile[0]: flist = glob(args.inputFile[0]) else: flist = args.inputFile elif args.inputFiles: if len(args.inputFiles) == 1 and args.inputFiles[0].endswith('.txt'): with open(args.inputFiles[0]) as f: flist = [l[:-1] for l in f.readlines()] else: flist = args.inputFiles else: fdefault = os.environ[ 'CMSSW_BASE'] + '/src/ntuplizer/BPH_RDntuplizer/production/' # fdefault += 'inputFiles_BP_Tag_B0_MuNuDmst_Hardbbbar_evtgen_ISGW2_PUc0_10-2-3.txt' fdefault += 'inputFiles_BP_Tag-Probe_B0_JpsiKst_Hardbbbar_evtgen_HELAMP_PUc0_10-2-3.txt' with open(fdefault) as f: flist = [l[:-1] for l in f.readlines()]
def contain_img(d): return len(glob(join(d, '*.jpg'))) > 0
# -*- coding: utf-8 -*- from pkg_resources import get_distribution, DistributionNotFound from os.path import dirname, basename, isfile import glob modules = glob.glob(dirname(__file__)+"/*.py") __all__ = [basename(f)[:-3] for f in modules if isfile(f) and not f.endswith('__init__.py')] name = "uncertainty_forest" try: # Change here if project is renamed and does not equal the package name dist_name = 'uncertainty-forest' __version__ = get_distribution(dist_name).version except DistributionNotFound: __version__ = 'unknown' finally: del get_distribution, DistributionNotFound