def create(self): """ Create a new environment """ output = Argv( self.conda, "create", "--yes", "--mkdir", "--prefix", self.path, "python={}".format(self.python), ).get_output(stderr=DEVNULL) match = re.search( r"\W*(.*activate) ({})".format(re.escape(str(self.path))), output ) self.source = self.pip.source = ( tuple(match.group(1).split()) + (match.group(2),) if match else ("activate", self.path) ) conda_env = Path(self.conda).parent.parent / 'etc' / 'profile.d' / 'conda.sh' if conda_env.is_file() and not is_windows_platform(): self.source = self.pip.source = CommandSequence(('source', conda_env.as_posix()), self.source) # install cuda toolkit try: cuda_version = float(int(self.session.config['agent.cuda_version'])) / 10.0 if cuda_version > 0: self._install('cudatoolkit={:.1f}'.format(cuda_version)) except Exception: pass return self
def saveMarketData(self, p_file_path="", p_save_type="CSV"): # Save the file to CSV my_file = Path(p_file_path) if my_file.is_file(): # File exists with open(p_file_path, 'a') as myfile: self.market_data.to_csv(myfile, header=False, date_format='%Y-%m-%d %H:%M:%S') # wr = csv.writer(myfile) # Add a empty row to ensure the CSV is written starting on the next row # Reverse it so it is date ascending (newest at the end) # print(len(self.market_data)) # wr.writerows(list(reversed(self.market_data))) else: # Create directory paths os.makedirs(os.path.dirname(p_file_path), exist_ok=True) # No file so create new file with open(p_file_path, 'w') as myfile: self.market_data.to_csv(p_file_path, header=False, date_format='%Y-%m-%d %H:%M:%S') # with open(p_file_path, 'w') as myfile: # wr = csv.writer(myfile) # # Reverse it so it is date ascending (newest at the end) # wr.writerows(reversed(self.market_data)) return
def initialize_logging(self): logging_config = self._config.get("logging", None) if not logging_config: return False # handle incomplete file handlers deleted = [] handlers = logging_config.get("handlers", {}) for name, handler in list(handlers.items()): cls = handler.get("class", None) is_file = cls and "FileHandler" in cls if cls is None or (is_file and "filename" not in handler): deleted.append(name) del handlers[name] elif is_file: file = Path(handler.get("filename")) if not file.is_file(): file.parent.mkdir(parents=True, exist_ok=True) file.touch() # remove dependency in deleted handlers root_logger = logging_config.get("root", None) loggers = list(logging_config.get( "loggers", {}).values()) + ([root_logger] if root_logger else []) for logger in loggers: handlers = logger.get("handlers", None) if not handlers: continue logger["handlers"] = [h for h in handlers if h not in deleted] extra = None if self._app: extra = {"app": self._app} initialize_log(logging_config, extra=extra) return True
def import_datasources(path: str, sync: str, recursive: bool) -> None: """Import datasources from YAML""" from superset.utils import dict_import_export sync_array = sync.split(",") path_object = Path(path) files = [] if path_object.is_file(): files.append(path_object) elif path_object.exists() and not recursive: files.extend(path_object.glob("*.yaml")) files.extend(path_object.glob("*.yml")) elif path_object.exists() and recursive: files.extend(path_object.rglob("*.yaml")) files.extend(path_object.rglob("*.yml")) for file_ in files: logger.info("Importing datasources from file %s", file_) try: with file_.open() as data_stream: dict_import_export.import_from_dict( yaml.safe_load(data_stream), sync=sync_array) except Exception as ex: # pylint: disable=broad-except logger.error("Error when importing datasources from file %s", file_) logger.error(ex)
def get_IB_data(ID): # Returns two namedtuples, with IB training and testing data # trn.X is training data # trn.y is trainiing class, with numbers from 0 to 9 # trn.Y is training class, but coded as a 10-dim vector with one entry set to 1 # similarly for tst nb_classes = 2 data_file = Path('datasets/IB_data_' + str(ID) + '.npz') if data_file.is_file(): data = np.load('datasets/IB_data_' + str(ID) + '.npz') else: create_IB_data(ID) data = np.load('datasets/IB_data_' + str(ID) + '.npz') (X_train, y_train), (X_test, y_test) = (data['X_train'], data['y_train']), (data['X_test'], data['y_test']) Y_train = keras.utils.np_utils.to_categorical(y_train, nb_classes).astype('float32') Y_test = keras.utils.np_utils.to_categorical(y_test, nb_classes).astype('float32') Dataset = namedtuple('Dataset', ['X', 'Y', 'y', 'nb_classes']) trn = Dataset(X_train, Y_train, y_train, nb_classes) tst = Dataset(X_test, Y_test, y_test, nb_classes) del X_train, X_test, Y_train, Y_test, y_train, y_test return trn, tst
def create_dbsqlite(): path = os.path.join(ROOT_PATH, "db.sqlite") dbsqlite = Path(path) if not dbsqlite.is_file(): with open(path, "wb"): os.utime(path, None)
def report_offline_session(cls, task, folder): filename = Path(folder) / cls.__offline_filename if not filename.is_file(): return False with open(filename.as_posix(), 'rt') as f: i = 0 while True: try: line = f.readline() if not line: break list_requests = json.loads(line) for r in list_requests: r.pop('task', None) i += 1 except StopIteration: break except Exception as ex: warning('Failed reporting log, line {} [{}]'.format(i, ex)) batch_requests = events.AddBatchRequest( requests=[events.TaskLogEvent(task=task.id, **r) for r in list_requests]) if batch_requests.requests: res = task.session.send(batch_requests) if res and not res.ok(): warning("failed logging task to backend ({:d} lines, {})".format( len(batch_requests.requests), str(res.meta))) return True
def _get_conda_sh(self): # type () -> Path base_conda_env = Path( self.conda).parent.parent / 'etc' / 'profile.d' / 'conda.sh' if base_conda_env.is_file(): return base_conda_env for path in os.environ.get('PATH', '').split( select_for_platform(windows=';', linux=':')): conda = find_executable("conda", path=path) if not conda: continue conda_env = Path( conda).parent.parent / 'etc' / 'profile.d' / 'conda.sh' if conda_env.is_file(): return conda_env return base_conda_env
def search(request): updatedb(request) if request.method == 'POST': form = ResultSearch(request.POST) if form.is_valid(): # Path.objects.filter(idsha256__contains=) sha256idquery = form.cleaned_data['id'] # queryid=Path.objects.get(idsha256=sha256id) try: pathdata = Path.objects.get(idsha256=sha256idquery) # Ok print pathdata.pathto print pathdata.idsha256 checkfile = Paths(pathdata.pathto + '/reports/report.json') if checkfile.is_file(): context = dashboardfunc(pathdata.pathto + '/reports/report.json') return render(request, 'analisys.html', context) else: messages.add_message(request, messages.INFO, 'ID not found') return redirect(request.META['HTTP_REFERER']) except ObjectDoesNotExist: messages.add_message(request, messages.INFO, 'ID not found') return redirect(request.META['HTTP_REFERER']) else: return render(request, 'search.html')
def resolve_local(document_loader, uri): pathpart, frag = urllib.parse.urldefrag(uri) pathobj = Path(pathpart).resolve() if pathobj.is_file(): if frag: return "{}#{}".format(pathobj.as_uri(), frag) else: return pathobj.as_uri() sharepaths = [ os.environ.get( "XDG_DATA_HOME", os.path.join(os.path.expanduser('~'), ".local", "share")) ] sharepaths.extend( os.environ.get("XDG_DATA_DIRS", "/usr/local/share/:/usr/share/").split(":")) shares = [os.path.join(s, "commonwl", uri) for s in sharepaths] _logger.debug("Search path is %s", shares) for path in shares: if os.path.exists(path): return Path(uri).as_uri() if os.path.exists("{}.cwl".format(path)): return Path("{}.cwl".format(path)).as_uri() return None
def list(request): if request.method == 'GET': dirtofile = request.GET.get('dir', '') if dirtofile != '': try: pathquery = dirtofile pathdata = Path.objects.get(pathto=pathquery) # Ok print pathdata.pathto print pathdata.idsha256 pathfile = pathdata.pathto + 'reports/report.json' checkfile = Paths(pathfile) if checkfile.is_file(): context = dashboardfunc(pathfile) return render(request, 'analisys.html', context) else: #filenotready = "File has not finished analysis" #query = Path.objects.all() #return render(request, 'list.html', {'filenotready': filenotready, 'query': query}) messages.add_message(request, messages.INFO, 'File has not finished analysis') return redirect(request.META['HTTP_REFERER']) except ObjectDoesNotExist: #filenotready = "File has not finished analysis" #query = Path.objects.all() #return render(request, 'list.html', {'filenotready': filenotready, 'query': query}) messages.add_message(request, messages.INFO, 'File has not finished analysis') return redirect(request.META['HTTP_REFERER']) else: updatedb(request) query = Path.objects.all() for i in query: print i.namefile return render(request, 'list.html', {'query': query})
def updatedb(request): listoffiles = glob.glob("/home/fernando/.cuckoo/storage/analyses/*") for i in listoffiles: if "latest" not in i: checkfile = Paths(i + "/reports/report.json") if checkfile.is_file(): with open(i + "/reports/report.json") as data_file: try: pathdata = Path.objects.get(pathto=i) data = json.load(data_file) filename = data['target']['file']['name'] file_sha = data['target']['file']['sha256'] dirpath = i + "/" pathdata.namefile = filename pathdata.idsha256 = file_sha pathdata.pathto = dirpath pathdata.save() except ObjectDoesNotExist: data = json.load(data_file) filename = data['target']['file']['name'] file_sha = data['target']['file']['sha256'] dirpath = i + "/" newinfo = Path(namefile=filename, idsha256=file_sha, pathto=dirpath) newinfo.save() return "1"
def get_IB_data(ID): # Returns two namedtuples, with IB training and testing data # trn.X is training data # trn.y is trainiing class, with numbers from 0 to 1 # trn.Y is training class, but coded as a 2-dim vector with one entry set to 1 # similarly for tst nb_classes = 2 data_file = Path('datasets/IB_data_'+str(ID)+'.npz') if data_file.is_file(): data = np.load('datasets/IB_data_'+str(ID)+'.npz') else: create_IB_data(ID) data = np.load('datasets/IB_data_'+str(ID)+'.npz') X_train, y_train= torch.FloatTensor(data['X_train']), torch.LongTensor(data['y_train'].squeeze()) X_test, y_test = torch.FloatTensor(data['X_test']), torch.LongTensor(data['y_test'].squeeze()) # y_train (0.8*4096, 1) y_test (0.2*4096,1) Y_train = one_hot_embedding(y_train, nb_classes) Y_test = one_hot_embedding(y_test, nb_classes) Dataset = namedtuple('Dataset',['X','Y','y','nb_classes']) trn = Dataset(X_train, Y_train, y_train, nb_classes) tst = Dataset(X_test , Y_test, y_test, nb_classes) del X_train, X_test, Y_train, Y_test, y_train, y_test return trn, tst
def handle(cls, name, args): # type: (str, Dict[str, Any]) -> None """Perform the actual test. Relies on .cfnlintrc file to be located beside the Runway config file. """ cfnlintrc = Path('./.cfnlintrc') if not cfnlintrc.is_file(): LOGGER.error('File must exist to use this test: %s', cfnlintrc) sys.exit(1) # prevent duplicate log messages by not passing to the root logger logging.getLogger('cfnlint').propagate = False try: with argv(*['cfn-lint'] + args.get('cli_args', [])): runpy.run_module('cfnlint', run_name='__main__') except SystemExit as err: # this call will always result in SystemExit if err.code != 0: # ignore zero exit codes but re-raise for non-zero if not (yaml.safe_load(cfnlintrc.read_text()) or {}).get('templates'): LOGGER.warning('cfnlintrc is missing a "templates" ' 'section which is required by cfn-lint') raise
def resolve_local(document_loader, uri): # type: (Loader, Text) -> Optional[Text] pathpart, frag = urllib.parse.urldefrag(uri) try: pathobj = Path(pathpart).resolve() except (WindowsError, OSError): _logger.debug("local resolver could not resolve %s", uri) return None if pathobj.is_file(): if frag: return "{}#{}".format(pathobj.as_uri(), frag) return pathobj.as_uri() sharepaths = [ os.environ.get( "XDG_DATA_HOME", os.path.join(os.path.expanduser('~'), ".local", "share")) ] sharepaths.extend( os.environ.get("XDG_DATA_DIRS", "/usr/local/share/:/usr/share/").split(":")) shares = [os.path.join(s, "commonwl", uri) for s in sharepaths] _logger.debug("Search path is %s", shares) for path in shares: if os.path.exists(path): return Path(uri).as_uri() if os.path.exists("{}.cwl".format(path)): return Path("{}.cwl".format(path)).as_uri() return None
def _check_and_get_file_obj(fpath): p = Path(fpath) if not p.parent.exists(): p.parent.mkdir(parents=True) if p.is_file(): return p.open('ab') return p.open('wb')
def make_gdb_cmd(prog_full_path, crashed_pid): """Construct a command that uses the POSIX debugger (gdb) to turn a minidump file into a stack trace. Args: prog_full_path (Path): Full path to the program crashed_pid (int): PID of the program Returns: list: gdb command list """ assert os.name == "posix" # On Mac and Linux, look for a core file. core_name = None if platform.system() == "Darwin": # Core files will be generated if you do: # mkdir -p /cores/ # ulimit -c 2147483648 (or call resource.setrlimit from a preexec_fn hook) core_name = "/cores/core." + str(crashed_pid) elif platform.system() == "Linux": is_pid_used = False core_uses_pid_path = Path("/proc/sys/kernel/core_uses_pid") if core_uses_pid_path.is_file(): with io.open(str(core_uses_pid_path), "r", encoding="utf-8", errors="replace") as f: is_pid_used = bool(int( f.read()[0])) # Setting [0] turns the input to a str. core_name = "core." + str(crashed_pid) if is_pid_used else "core" core_name_path = Path.cwd() / core_name if not core_name_path.is_file(): core_name_path = Path.home() / core_name # try the home dir if core_name and core_name_path.is_file(): dbggr_cmd_path = Path(__file__).parent / "gdb_cmds.txt" assert dbggr_cmd_path.is_file() # pylint: disable=no-member # Run gdb and move the core file. Tip: gdb gives more info for: # (debug with intact build dir > debug > opt with frame pointers > opt) return [ "gdb", "-n", "-batch", "-x", str(dbggr_cmd_path), str(prog_full_path), str(core_name) ] return []
def get_file(path): result = Path('web') / path if result.is_file(): return str(result) if result.is_dir() and (result / 'index.html').is_file(): return str(result / 'index.html') # File was not found. return None
def make_cdb_cmd(prog_full_path, crashed_pid): """Construct a command that uses the Windows debugger (cdb.exe) to turn a minidump file into a stack trace. Args: prog_full_path (Path): Full path to the program crashed_pid (int): PID of the program Returns: list: cdb command list """ assert platform.system() == "Windows" # Look for a minidump. dump_name = Path.home() / "AppData" / "Local" / "CrashDumps" / ( "%s.%s.dmp" % (prog_full_path.name, crashed_pid)) if platform.uname()[2] == "10": # Windows 10 win64_debugging_folder = Path(os.getenv( "PROGRAMFILES(X86)")) / "Windows Kits" / "10" / "Debuggers" / "x64" else: win64_debugging_folder = Path( os.getenv("PROGRAMW6432")) / "Debugging Tools for Windows (x64)" # 64-bit cdb.exe seems to also be able to analyse 32-bit binary dumps. cdb_path = win64_debugging_folder / "cdb.exe" if not cdb_path.is_file(): # pylint: disable=no-member print() print( "WARNING: cdb.exe is not found - all crashes will be interesting.") print() return [] if is_win_dumping_to_default(): loops = 0 max_loops = 300 while True: if dump_name.is_file(): dbggr_cmd_path = Path(__file__).parent / "cdb_cmds.txt" assert dbggr_cmd_path.is_file() # pylint: disable=no-member cdb_cmd_list = [] cdb_cmd_list.append("$<" + str(dbggr_cmd_path)) # See bug 902706 about -g. return [ cdb_path, "-g", "-c", ";".join(cdb_cmd_list), "-z", str(dump_name) ] time.sleep(0.200) loops += 1 if loops > max_loops: # Windows may take some time to generate the dump. print( "make_cdb_cmd waited a long time, but %s never appeared!" % str(dump_name)) return [] else: return []
def _make_file_info(target: pathlib.Path, arcname: Optional[str] = None) -> Dict[str, Any]: f = {} # type: Dict[str, Any] f['origin'] = str(target) if arcname is not None: f['filename'] = arcname else: f['filename'] = str(target) if os.name == 'nt': fstat = os.stat(str(target), follow_symlinks=False) if target.is_symlink(): f['emptystream'] = False f['attributes'] = fstat.st_file_attributes & FILE_ATTRIBUTE_WINDOWS_MASK # type: ignore # noqa elif target.is_dir(): f['emptystream'] = True f['attributes'] = fstat.st_file_attributes & FILE_ATTRIBUTE_WINDOWS_MASK # type: ignore # noqa elif target.is_file(): f['emptystream'] = False f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa f['uncompressed'] = fstat.st_size else: fstat = target.stat() if target.is_symlink(): f['emptystream'] = False f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | ( stat.S_IFLNK << 16) f['attributes'] |= (stat.S_IMODE(fstat.st_mode) << 16) elif target.is_dir(): f['emptystream'] = True f['attributes'] = stat.FILE_ATTRIBUTE_DIRECTORY # type: ignore # noqa f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | ( stat.S_IFDIR << 16) f['attributes'] |= (stat.S_IMODE(fstat.st_mode) << 16) elif target.is_file(): f['emptystream'] = False f['uncompressed'] = fstat.st_size f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | ( stat.S_IMODE(fstat.st_mode) << 16) f['creationtime'] = target.stat().st_ctime f['lastwritetime'] = target.stat().st_mtime f['lastaccesstime'] = target.stat().st_atime return f
def _get_script_info(cls, filepath, check_uncommitted=False, log=None): jupyter_filepath = cls._get_jupyter_notebook_filename() if jupyter_filepath: script_path = Path(os.path.normpath(jupyter_filepath)).absolute() else: script_path = Path(os.path.normpath(filepath)).absolute() if not script_path.is_file(): raise ScriptInfoError( "Script file [{}] could not be found".format(filepath)) script_dir = script_path.parent def _log(msg, *args, **kwargs): if not log: return log.warning("Failed auto-detecting task repository: {}".format( msg.format(*args, **kwargs))) plugin = next((p for p in cls.plugins if p.exists(script_dir)), None) repo_info = DetectionResult() if not plugin: _log("expected one of: {}", ", ".join( (p.name for p in cls.plugins))) else: try: repo_info = plugin.get_info(str(script_dir), include_diff=check_uncommitted) except Exception as ex: _log("no info for {} ({})", script_dir, ex) else: if repo_info.is_empty(): _log("no info for {}", script_dir) repo_root = repo_info.root or script_dir working_dir = cls._get_working_dir(repo_root) entry_point = cls._get_entry_point(repo_root, script_path) script_info = dict( repository=furl(repo_info.url).remove(username=True, password=True).tostr(), branch=repo_info.branch, version_num=repo_info.commit, entry_point=entry_point, working_dir=working_dir, diff=repo_info.diff, ) messages = [] if repo_info.modified: messages.append( "======> WARNING! UNCOMMITTED CHANGES IN REPOSITORY {} <======" .format(script_info.get("repository", ""))) if not any(script_info.values()): script_info = None return ScriptInfoResult(script=script_info, warning_messages=messages)
def checkTLE(self, index): """Check if TLE exists for given satellite, else call getnewtle() Args: index (str): NORAD ID of satellite """ TLEfileExists = Path("./TLE/" + index + '.txt') if (TLEfileExists.is_file() == False): self.getnewtle(index)
def test_preprocess_saves_result_in_the_right_folder(path_to_config, make_tmp_folder): yass.set_config(path_to_config, make_tmp_folder) standarized_path, standarized_params, _ = preprocess.run() expected = Path(make_tmp_folder, 'preprocess', 'standarized.bin') assert str(expected) == standarized_path assert expected.is_file()
def is_resource(package, name): """True if name is a resource inside package. Directories are *not* resources. """ package = _get_package(package) _normalize_path(name) try: package_contents = set(contents(package)) except OSError as error: if error.errno not in (errno.ENOENT, errno.ENOTDIR): # We won't hit this in the Python 2 tests, so it'll appear # uncovered. We could mock os.listdir() to return a non-ENOENT or # ENOTDIR, but then we'd have to depend on another external # library since Python 2 doesn't have unittest.mock. It's not # worth it. raise # pragma: nocover return False if name not in package_contents: return False # Just because the given file_name lives as an entry in the package's # contents doesn't necessarily mean it's a resource. Directories are not # resources, so let's try to find out if it's a directory or not. path = Path(package.__file__).parent / name if path.is_file(): return True if path.is_dir(): return False # If it's not a file and it's not a directory, what is it? Well, this # means the file doesn't exist on the file system, so it probably lives # inside a zip file. We have to crack open the zip, look at its table of # contents, and make sure that this entry doesn't have sub-entries. archive_path = package.__loader__.archive # type: ignore package_directory = Path(package.__file__).parent with ZipFile(archive_path) as zf: toc = zf.namelist() relpath = package_directory.relative_to(archive_path) candidate_path = relpath / name for entry in toc: # pragma: nobranch try: relative_to_candidate = Path(entry).relative_to(candidate_path) except ValueError: # The two paths aren't relative to each other so we can ignore it. continue # Since directories aren't explicitly listed in the zip file, we must # infer their 'directory-ness' by looking at the number of path # components in the path relative to the package resource we're # looking up. If there are zero additional parts, it's a file, i.e. a # resource. If there are more than zero it's a directory, i.e. not a # resource. It has to be one of these two cases. return len(relative_to_candidate.parts) == 0 # I think it's impossible to get here. It would mean that we are looking # for a resource in a zip file, there's an entry matching it in the return # value of contents(), but we never actually found it in the zip's table of # contents. raise AssertionError('Impossible situation')
def generate_new_csv_overall(): my_file = Path("./stats/instaPyStats.csv") if my_file.is_file(): return 'The overall File already exists' else: filename = "./stats/instaPyStats.csv" with open(filename, mode='a') as insta_file: stat_writer = csv.writer(insta_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) stat_writer.writerow(['DATE', 'USER_NAME', 'FOLLOWERS', 'FOLLOWING']) return "New overall history file created"
def recursive_search(path): # print(path) ign_flag = 0 ignore_file = Path(os.path.join(path, ".ignore")) if ignore_file.is_file(): ign_flag = 1 text_file.write("# ignored - " + path + "\n") # local set skip_node_flag = 0 skip_node_file = Path(os.path.join(path, ".ignore_node")) if skip_node_file.is_file(): skip_node_flag = 1 text_file.write("# ignored directory and all subdirs - " + path + "\n") return None local_source_files = set() for file in os.listdir(path): if (file == ".") or (file == ".."): continue full_name = os.path.join(path, file) if os.path.isdir(full_name) and not os.path.islink(path): recursive_search(full_name) if ign_flag == 0: if re.search("\\.(c|cpp|cxx)$", file) is not None: local_source_files.add(full_name) # local_source_files.add(file) elif re.search("\\.(h|hpp)$", file) is not None: include_dirs.add(path) if len(local_source_files) != 0: # dir_var = "set(DIR_" + path + ")" # print(dir_var) path_g = path[skip_chars:] path_g = path_g.replace("/", "_") path_g = path_g.replace("\\", "_") group_name = "GROUP_" + path_g.upper() source_files.add(group_name) text_file.write("\nset(" + group_name + "\n") for fname in sorted(local_source_files): fname = fname[skip_chars:].replace("\\", "/") text_file.write("\t\t" + fname + "\n") text_file.write(")" + "\n")
def import_config(): """import the configuration for the dimension reduction in this file """ module_dir = os.path.dirname(os.path.realpath(__file__)) running_dir = os.getcwd() module_file = Path(os.path.join(module_dir, "algorithms.json")) running_file = Path(os.path.join(running_dir, "algorithms.json")) def rfile(f): with f.open() as param: config_ = json.load(param) return config_ if running_file.is_file(): return rfile(running_file) elif module_file.is_file(): return rfile(module_file) else: raise IOError, "No such file 'algorithms.json'"
def generate_new_csv(usr_name): my_file = Path("./stats/" + usr_name + ".csv") if my_file.is_file(): return 'The history file for "' + usr_name + '" already exists' else: filename = './stats/' + usr_name + '.csv' with open(filename, mode='a') as insta_file: stat_writer = csv.writer(insta_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) stat_writer.writerow(['DATE', 'FOLLOWERS', 'FOLLOWING']) return "New history file created for user: " + usr_name
def update_model_desc(self, new_model_desc_file=None): """ Change the task's model_desc """ execution = self._get_task_property('execution') p = Path(new_model_desc_file) if not p.is_file(): raise IOError('mode_desc file %s cannot be found' % new_model_desc_file) new_model_desc = p.read_text() model_desc_key = list(execution.model_desc.keys())[0] if execution.model_desc else 'design' execution.model_desc[model_desc_key] = new_model_desc res = self._edit(execution=execution) return res.response
def getFilesFromFolderByFiletype(folder_selected, filetype): files_from_folder = [] for root, dirs, files in os.walk(folder_selected): for file in files: if file.endswith(filetype): input_file = os.path.join(root, file) output_file = Path(getOutputFileFromInputFile(input_file)) if not output_file.is_file(): files_from_folder.append(input_file) return files_from_folder
def parse_file(fname): aminoacids = [] the_file = Path(fname) if the_file.is_file(): with gzip.open(fname, 'rb') as fin: for line in fin: if query_ca.match(line): atname, resname, resid, xyz = atom_from_string(line) aminoacids.append(AlphaCarbon(resname, xyz)) elif line.startswith('TER'): aminoacids.append(None) return aminoacids
def cmd_run(path): """ Runs an appliction. """ os.chdir(path) package = Path("./package.json") if not package.is_file(): raise Exception("Invalid package: no package.json file") package = json.load(package.open()) if "engines" not in package or package["engines"] == {}: raise Exception("Invalid package: no engines specified") r = requests.get("%s/index.json" % Particle.REPO) r.raise_for_status() remote_particles = r.json()["particles"] variables = {} for name, range_ in package["engines"].items(): p = Particle.get_local(name, range_) if not p: # if auto_fetch: if name in remote_particles: v = semver.max_satisfying(remote_particles[name], range_, False) if v: print("Downloading %s %s..." % (name, v)) p = Particle.fetch(name, v) else: print("Cannot satisfy %s (%s), aborting." % (name, range_)) sys.exit(1) else: print("No particle named %s exists, aborting." % name) sys.exit(1) variables["$" + name.upper().replace("-", "_")] = str(p.main) pattern = re.compile('|'.join(map(re.escape, variables.keys()))) if "lepton" not in package: raise Exception("Invalid package: no lepton key in particle.json") elif "run" not in package["lepton"]: raise Exception("Invalid package: no lepton.run key in particle.json") args = package["lepton"]["run"] args = pattern.sub(lambda x: variables[x.group()], args) args = shlex.split(args) print("Resulting command line: %r" % args) print("Current dir: %s" % os.getcwd()) os.execvp(args[0], args)
def import_dashboards(path, recursive=False): """Import dashboards from JSON""" p = Path(path) files = [] if p.is_file(): files.append(p) elif p.exists() and not recursive: files.extend(p.glob('*.json')) elif p.exists() and recursive: files.extend(p.rglob('*.json')) for f in files: logging.info('Importing dashboard from file %s', f) try: with f.open() as data_stream: dashboard_import_export.import_dashboards( db.session, data_stream) except Exception as e: logging.error('Error when importing dashboard from file %s', f) logging.error(e)
def get_file(path): filepath = Path('site') / path if filepath.is_file(): return filepath for format in PAGE_FORMATS: index_file = filepath / ('index' + format) if filepath.is_dir() and index_file.is_file(): return index_file if filepath.suffix == '.css': style_file = filepath.parent / (filepath.stem + '.styl') if style_file.exists(): return style_file if filepath.suffix == '.js': coffee_file = filepath.parent / (filepath.stem + '.coffee') if coffee_file.exists(): return coffee_file return None
def resolve_local(document_loader, uri): pathpart, frag = urllib.parse.urldefrag(uri) pathobj = Path(pathpart).resolve() if pathobj.is_file(): if frag: return "{}#{}".format(pathobj.as_uri(), frag) return pathobj.as_uri() sharepaths = [os.environ.get("XDG_DATA_HOME", os.path.join( os.path.expanduser('~'), ".local", "share"))] sharepaths.extend(os.environ.get( "XDG_DATA_DIRS", "/usr/local/share/:/usr/share/").split(":")) shares = [os.path.join(s, "commonwl", uri) for s in sharepaths] _logger.debug("Search path is %s", shares) for path in shares: if os.path.exists(path): return Path(uri).as_uri() if os.path.exists("{}.cwl".format(path)): return Path("{}.cwl".format(path)).as_uri() return None
def Deepzoom(image_or_directory_path,create_static_cache=False,**kwargs): """ Returns a Deepzoom interface corresponding to the given image. Can accept either a filepath (and will read tiles on the fly) or a directory path (that contains an image with the same name as the directory) that contains/will contain a static DeepZoom image directory. :param image_or_directory_path: String or Pathlib object :param create_static_cache: If True, creates a static DeepZoom image directory directory structure *around* the given image (or in the given directory). This is done lazily, saving each tile as it's requested. :param kwargs: Same as DeepzoomInterface :return: DeepZoom """ p = Path(image_or_directory_path).resolve() img = None if p.is_file(): img = _ImageFactory(p) elif p.is_dir(): fList = list(p.glob('%s.*'%p.name)) if len(fList)==0: raise IOError('Invalid Deepzoom directory (%s). ' 'Must contain and image named (%s) to be valid.' ''%(p,'%s.<EXT>'%p.name)) for f in fList: try: img = _ImageFactory(f) except IOError: pass if img is None: raise IOError('Invalid Deepzoom target (%s). ' 'Not a supported image format.'%(p)) if create_static_cache: # do something to DeepZoomGenerator so that it saves on get_tile() dzGen = _CachedInterface(img,**kwargs) else: dzGen = _DeepzoomInterface(img,**kwargs) return dzGen
def import_datasources(path, sync, recursive=False): """Import datasources from YAML""" sync_array = sync.split(',') p = Path(path) files = [] if p.is_file(): files.append(p) elif p.exists() and not recursive: files.extend(p.glob('*.yaml')) files.extend(p.glob('*.yml')) elif p.exists() and recursive: files.extend(p.rglob('*.yaml')) files.extend(p.rglob('*.yml')) for f in files: logging.info('Importing datasources from file %s', f) try: with f.open() as data_stream: dict_import_export_util.import_from_dict( db.session, yaml.safe_load(data_stream), sync=sync_array) except Exception as e: logging.error('Error when importing datasources from file %s', f) logging.error(e)
def test_has_files(tmp_crumb): assert not op.exists(tmp_crumb.path) assert not tmp_crumb.has_files() values_dict = {'session_id': ['session_{:02}'.format(i) for i in range( 2)], 'subject_id': ['subj_{:03}'.format(i) for i in range( 3)], 'modality': ['anat'], 'image': ['mprage1.nii', 'mprage2.nii', 'mprage3.nii'], } paths = mktree(tmp_crumb, list(ParameterGrid(values_dict))) assert op.exists(tmp_crumb.split()[0]) assert not tmp_crumb.has_files() pa = Path(str(paths[0])) pa.rmdir() pa.touch() assert pa.exists() assert pa.is_file() assert tmp_crumb.has_files()
def cmd_run(path): """ Runs an appliction. """ os.chdir(path) package = Path("./package.json") if not package.is_file(): raise Exception("Invalid package: no package.json file") package = json.load(package.open()) if "engines" not in package or package["engines"] == {}: raise Exception("Invalid package: no engines specified") variables = {} for name, version in package["engines"].items(): p = Particle.get_local(name, version) if not p: # if auto_fetch: print("Downloading %s..." % name) p = Particle.fetch(name, version) variables["$" + name.upper().replace("-", "_")] = str(p.main) pattern = re.compile('|'.join(map(re.escape, variables.keys()))) if "lepton" not in package: raise Exception("Invalid package: no lepton key in particle.json") elif "run" not in package["lepton"]: raise Exception("Invalid package: no lepton.run key in particle.json") args = package["lepton"]["run"] args = pattern.sub(lambda x: variables[x.group()], args) args = shlex.split(args) print("Resulting command line: %r" % args) print("Current dir: %s" % os.getcwd()) os.execvp(args[0], args)