def test_make_directories(mount_dir, sub_done, sub_open): with pytest.raises(FileExistsError): mkdir(sub_open, 'dir') assert isdir(sub_open, 'dir') with pytest.raises(PermissionError): rm_rf(sub_open, 'dir') assert isdir(sub_open, 'dir') assert isfile(sub_open, 'dir', 'single_file_work') assert isfile(sub_open, 'dir', 'single_file_work_copy') with pytest.raises(IsADirectoryError): rm(sub_done, 'dir') with pytest.raises(OSError): rmdir(sub_done, 'dir') assert isdir(sub_done, 'dir') rm_rf(sub_done, 'dir') assert 'dir' not in ls(sub_done) mkdir(sub_done, 'dir') assert isdir(sub_done, 'dir') rmdir(sub_done, 'dir') assert 'dir' not in ls(sub_done)
def test_compiling_program(sub_done, mount, fixed): url = 'https://attach.libremail.nl/__test_codegra.fs__.tar.gz' fname = join(sub_done, '42.tar.gz') fdir = join(sub_done, '42sh/') urllib.request.urlretrieve(url, fname) tar = tarfile.open(fname, "r:gz") tar.extractall(sub_done) tar.close() print(subprocess.check_output(['make', '-C', fdir])) assert isdir(fdir) assert isfile(fname) assert subprocess.check_output( [join(fdir, '42sh'), '-c', 'echo hello from 42'] ).decode('utf-8') == 'hello from 42\n' mount() if fixed: assert not isdir(fdir) assert not isfile(fname) return assert isfile(fname) assert isdir(fdir) assert subprocess.check_output( [join(fdir, '42sh'), '-c', 'echo hello from 42'] ).decode('utf-8') == 'hello from 42\n' rm_rf(fdir) assert not isdir(fdir) mount() assert isfile(fname) assert not isdir(fdir)
def test_invalid_perm_setting(sub_done): assert isfile(sub_done, 'dir', 'single_file_work') with pytest.raises(PermissionError): chown([sub_done, 'dir', 'single_file_work'], 7, 7) assert isfile(sub_done, 'dir', 'single_file_work') with pytest.raises(PermissionError): chmod([sub_done, 'dir', 'single_file_work'], stat.S_IRGRP) assert isfile(sub_done, 'dir', 'single_file_work')
def test_delete_files(mount_dir, sub_open, sub_done): with open(join(sub_open, 'file1'), 'w') as f: f.write('abc\n') assert isfile(sub_open, 'file1') with open(join(sub_open, 'file1'), 'r') as f: assert f.read() == 'abc\n' rm(sub_open, 'file1') assert not isfile(sub_open, 'file1') with pytest.raises(FileNotFoundError): rm(sub_open, 'nonexisting')
def test_renaming_file_in_fixed(sub_open, mount): fname = join(sub_open, 'new_test_file') fname2 = join(sub_open, 'new_test_file2') fname3 = join(sub_open, 'new_test_file3') # Make sure we cannot delete existing files in fixed mode assert not isfile(fname) with open(fname, 'w') as f: f.write('Hello thomas\n') assert isfile(fname) mount(fixed=True) assert isfile(fname) with pytest.raises(PermissionError): rename([fname], [fname2]) del fname assert not isfile(fname2) with open(fname2, 'w') as f: f.write('Hello thomas\n') assert isfile(fname2) rename([fname2], [fname3]) assert not isfile(fname2) assert isfile(fname3) with open(fname3, 'r') as f: assert f.read() == 'Hello thomas\n' mount(fixed=True) assert not isfile(fname3)
def test_editing_file_in_fixed(sub_open, mount): fname = join(sub_open, 'new_test_file') # Make sure we cannot delete existing files in fixed mode assert not isfile(fname) with open(fname, 'w') as f: f.write('Hello thomas\n') assert isfile(fname) mount(fixed=True) assert isfile(fname) with pytest.raises(PermissionError): with open(fname, 'w') as f: f.write('hello\n')
def __load(self): if isfile(self.file): with open(self.file, 'rb') as fp: self.data = pickle.load(fp) if 'items' not in self.data or 'file' not in self.data: raise FileExistsError( 'Could not resume due to corrupted track file, please remove the file and try again: %s' % self.file) if not isfile(self.data['file']): raise FileNotFoundError( 'Previous output file is missing, unable to resume: %s' % self.data['file']) self._oldout = self.data['file'] else: self.data = {'items': set(), 'file': self._newout}
def probe_cache(cid, fpath='probe_cache/'): global cached_probes global fcached_probes fnum = cid / 1000 fpath = format_dirpath(fpath) fname = fpath+'pc'+str(fnum)+'.json' ret = None scid = str(cid) if isfile(fname): with open(fname, 'r+') as f: pc = json.load(f) if scid in pc: ret = pc[scid] if ret is None: pc = dict() try: tmp = Probe(id=cid) pc[scid] = {'country': tmp.country_code, 'asn': tmp.asn_v4, 'prefix': tmp.prefix_v4, 'coords': tmp.geometry['coordinates']} ret = pc[scid] except Exception as e: print(e) print('failed to get '+scid) if scid in pc: with open(fname, 'w+') as f: json.dump(pc, f) with open(fcached_probes, 'w+') as f: json.dump(list(cached_probes), f) cached_probes.add(cid) if ret is None: raise Exception('failed to get client info') else: return ret
def __add_header(self): if not isfile(self.file): if self.firstrow > 0: for i in range(self.firstrow): self.append([]) if self.header is not None: self.append(self.header)
def parse(fpath): """ Routes what parser to use depending on the file extension of a file :param fpath: file path of file :return: none """ # Check existence if not isfile(fpath): raise FileNotFoundError('File is not existed') # Get file extension ext = getext(fpath) # Get module parser = None for lang, exts in LANGUAGE_EXTENSIONS.items(): if ext in exts: parser = getattr(sys.modules[__name__], lang) break # If extension is not found if parser is None: raise FileExistsError('File extension is not supported') # Parse and return return parser(fpath)
def test_create_files(mount_dir, sub_open, sub_done): with pytest.raises(PermissionError): with open(join(sub_open, 'file1'), 'w') as f: pass assert not isfile(sub_open, 'file1') with pytest.raises(PermissionError): with open(join(sub_open, 'file1'), 'w') as f: f.write('abc\n') assert not isfile(sub_open, 'file1') with open(join(sub_done, 'file1'), 'w') as f: pass assert isfile(sub_done, 'file1') with open(join(sub_done, 'file2'), 'w') as f: f.write('abc\n') assert isfile(sub_done, 'file2')
def test_quote_paths(sub_done, path): dirs = path.split('/')[:-1] if len(dirs): mkdir(join(sub_done, *dirs)) f_path = join(sub_done, path) with open(join(sub_done, f_path), 'w') as f: assert f.write('abc') == 3 tmp_path = join(sub_done, 'abc') rename([f_path], [tmp_path]) assert not isfile(f_path) assert isfile(tmp_path) rename([tmp_path], [f_path]) assert isfile(f_path) assert not isfile(tmp_path) rm(f_path) if len(dirs): rmdir(join(sub_done, *dirs))
def test_bug_fsync(mount_dir, sub_done): # The `fsync` function didn't return the value of `flush` so editing a # teacher file didn't update the internal id of the file. p = join(sub_done, 'dir', 'single_file_work') assert isfile(p) with open(p, 'r') as f: old = f.read() new = 'NEW CONTENT' + old fd = os.open(p, os.O_WRONLY | os.O_TRUNC) os.write(fd, bytes(new, 'utf8')) os.fsync(fd) os.close(fd) with open(p, 'r') as f: assert new == f.read()
def test_write_to_directory(sub_done): fdir = join(sub_done, 'new_dir') fname = join(fdir, 'new_file') mkdir(fdir) assert isdir(fdir) assert not isfile(fdir) with pytest.raises(IsADirectoryError): with open(fdir, 'w') as f: f.write('hello\n') open(fname, 'w').close() with pytest.raises(NotADirectoryError): ls(fname) with pytest.raises(FileExistsError): mkdir(fname)
def test_write_and_read_files(mount_dir, sub_open, sub_done): assert isfile(sub_done, 'dir', 'single_file_work') with open(join(sub_done, 'dir', 'single_file_work'), 'w') as f: f.write('abc\n') assert isfile(sub_done, 'dir', 'single_file_work') with open(join(sub_done, 'dir', 'single_file_work'), 'r') as f: assert f.read() == 'abc\n' with open(join(sub_done, 'file1'), 'a') as f: f.write('abc\n') assert isfile(sub_done, 'file1') with open(join(sub_done, 'file1'), 'r') as f: assert f.read() == 'abc\n' with open(join(sub_done, 'file1'), 'w') as f: f.write('def\n') with open(join(sub_done, 'file1'), 'r') as f: assert f.read() == 'def\n' with open(join(sub_done, 'file1'), 'a') as f: f.write('def\n') with open(join(sub_done, 'file1'), 'r') as f: assert f.read() == 'def\ndef\n' with pytest.raises(PermissionError): with open(join(sub_open, 'file1'), 'a') as f: f.write('def\n') assert not isfile(sub_open, 'file1') assert isfile(sub_open, 'dir', 'single_file_work') with open(join(sub_open, 'dir', 'single_file_work'), 'r') as f: old = f.read() with pytest.raises(PermissionError): with open(join(sub_open, 'dir', 'single_file_work'), 'a') as f: f.write('abc\n') assert isfile(sub_open, 'dir', 'single_file_work') with open(join(sub_open, 'dir', 'single_file_work'), 'r') as f: assert f.read() == old
def get_probe_distances(data=None, procs=4, domtotal=302, fname0='mapped_probes.pkl'): print(inspect.stack()[0][3]) if isfile(fname0): data = pd.read_pickle(fname0) else:
def get_args(): parser = ArgumentParser(description=APP_DESC, prog=APP_CMD) parser.add_argument('-f', '--file', metavar='FILE', help='path to the PDF file that is to be parsed') parser.add_argument( '-d', '--dir', metavar='DIR', help= 'path to the directory containing the PDF files that are to be parsed') parser.add_argument('-s', '--state', metavar='STATE', help='state whose PDF rolls(s) are to be parsed') parser.add_argument('-l', '--lang', metavar='LANG', default='english', help='specify language used in parsing \ documents (default is English if not being specified)') parser.add_argument('-o', '--out', metavar='FILE', default=OUTPUT_FILE.format(timestamp(OUTPUT_FILE_TS)), help='\ specify output file for storing parsed result (must be \'.csv\' file). The default output file is \ \'Parsed-{timestamp}.csv\' and stored in the \'output\' directory') parser.add_argument('--resume', action='store_true', default=False, help='allows the parsing to be \ resumed later if the program is stopped unexpectedly or intentionally. Only takes effect if it is applied to a directory' ) parser.add_argument('--version', action='version', version=APP_NAME_WITH_VERSION) parser.add_argument('--all-states', action='store_true', default=False, help='show all the states that are supported and exit') args = parser.parse_args() if args.all_states: out = [] chars = max(len(x) for x in states.keys()) for state, langs in states.items(): out.append('%-{}s : %s'.format(chars) % (state, ', '.join(langs))) parser.exit(message='States supported:\n%s\n' % '\n'.join(out)) if not args.file and not args.dir or not args.state: parser.error( 'the following arguments are required: -f/--file or -d/--dir, -s/--state' ) if args.file and args.dir: parser.error('only accepts one of --file or --dir at a time') elif args.file: if not isfile(args.file): parser.error('file not found: %s' % args.file) if not ispdf(args.file): parser.error('file must be in PDF format') if args.resume: parser.error('parsing file does not allow --resume') elif args.dir: if not isdir(args.dir): parser.error('folder not found: %s' % args.dir) if args.state.lower() not in states: parser.error('unsupported state \'%s\'' % args.state) if args.lang and args.lang.lower() not in states[args.state.lower()]: parser.error('state \'%s\' does not have \'%s\' language' % (args.state, args.lang)) if dirname(args.out) and not isdir(dirname(args.out)): parser.error('output folder does not exist: %s' % dirname(args.out)) if filext(args.out).lower() != 'csv': parser.error('output file must be \'.csv\'') args.path = args.file or args.dir args.state = args.state.lower() args.lang = args.lang.lower() return args
def cleanup(self): if isfile(self.file): rmfile(self.file)
import pyasn from helpers import format_dirpath, isfile import json from ripe.atlas.cousteau import Probe from IPy import IP from collections import defaultdict asndb = pyasn.pyasn('asndb.dat') fcached_probes = 'cached_probes.list' import numpy as np if isfile(fcached_probes): with open(fcached_probes, 'r+') as f: cached_probes = set(json.load(f)) else: cached_probes = set() def is_public(i): try: ip = IP(i) return ip.iptype() == 'PUBLIC' except Exception as e: print(e) pass return False def get_prefix(ip, cid=-1): prefix = 24 try:
def test_deleting_file_in_fixed(sub_open, mount): fname = join(sub_open, 'new_test_file') fname2 = join(sub_open, 'new_test_file2') # Make sure we cannot delete existing files in fixed mode assert not isfile(fname) with open(fname, 'w') as f: f.write('Hello thomas\n') assert isfile(fname) mount(fixed=True) assert isfile(fname) with open(fname, 'r') as f: assert f.read() == 'Hello thomas\n' with pytest.raises(PermissionError): with open(fname, 'w') as f: f.write('Hello thomas\n') with pytest.raises(PermissionError): rm(fname) del fname # Now make sure we can delete files that did not exist assert not isfile(fname2) with open(fname2, 'w') as f: f.write('Hello thomas2\n') with open(fname2, 'r') as f: assert f.read() == 'Hello thomas2\n' assert isfile(fname2) rm(fname2) assert not isfile(fname2) # Make sure files created in fixed mode are not visible after a remount assert not isfile(fname2) with open(fname2, 'w') as f: f.write('Hello thomas2\n') with open(fname2, 'r') as f: assert f.read() == 'Hello thomas2\n' assert isfile(fname2) mount(fixed=True) assert not isfile(fname2) with open(fname2, 'w') as f: f.write('Hello thomas2\n') with open(fname2, 'r') as f: assert f.read() == 'Hello thomas2\n' assert isfile(fname2) mount(fixed=True) assert not isfile(fname2)
def main(): """Main program""" # Check Python if not minpy(MINIMUM_PYTHON_REQUIRED): print('Failed! Minimum Python required is %s' % MINIMUM_PYTHON_REQUIRED) sys.exit(1) # Read arguments try: args = readargs(**COMMAND_LINE_SETTINGS) except SyntaxError as msg: print(msg) sys.exit(2) except SystemExit: sys.exit() # Load settings wdir = args.get('dir', DEFAULT_WORKING_DIR) xml = joinroot(wdir, args.get('xmlfile', DEFAULT_XML_FILE)) out = joinroot(wdir, args.get('outfile', DEFAULT_OUTPUT_FILE)) lang = args.get('lang', None) anyext = True if 'anyext' in args else False # Check XML file existence if not isfile(xml): print('Failed! XML file is not existed:', xml) sys.exit(1) # Check output file existence if isfile(out): print('Output file is existed:', out) confirm = input( 'Do you want to overwrite? (type Y to confirm, any key to exit): ') if confirm != 'Y' and confirm != 'y': sys.exit() # Make sure output file is not a dir if isdir(out): print('Failed! Ouput file path is a directory:', out) sys.exit(1) # Check output file parent dir if not dirfound(out, path_is_file=True): print('Failed! Output file\'s parent dir is not found:', out) sys.exit(1) # Do generating try: MDGen(infile=xml, outfile=out, workdir=wdir, language=lang, any_ext=anyext) except Exception as msg: print('Failed! Error found: %s' % msg) sys.exit(2) # Done print('MD file successfully generated:', out)