def test_make_directories(mount_dir, sub_done, sub_open):
with pytest.raises(FileExistsError):
mkdir(sub_open, 'dir')
assert isdir(sub_open, 'dir')
with pytest.raises(PermissionError):
rm_rf(sub_open, 'dir')
assert isdir(sub_open, 'dir')
assert isfile(sub_open, 'dir', 'single_file_work')
assert isfile(sub_open, 'dir', 'single_file_work_copy')
with pytest.raises(IsADirectoryError):
rm(sub_done, 'dir')
with pytest.raises(OSError):
rmdir(sub_done, 'dir')
assert isdir(sub_done, 'dir')
rm_rf(sub_done, 'dir')
assert 'dir' not in ls(sub_done)
mkdir(sub_done, 'dir')
assert isdir(sub_done, 'dir')
rmdir(sub_done, 'dir')
assert 'dir' not in ls(sub_done)
def test_compiling_program(sub_done, mount, fixed):
url = 'https://attach.libremail.nl/__test_codegra.fs__.tar.gz'
fname = join(sub_done, '42.tar.gz')
fdir = join(sub_done, '42sh/')
urllib.request.urlretrieve(url, fname)
tar = tarfile.open(fname, "r:gz")
tar.extractall(sub_done)
tar.close()
print(subprocess.check_output(['make', '-C', fdir]))
assert isdir(fdir)
assert isfile(fname)
assert subprocess.check_output(
[join(fdir, '42sh'), '-c', 'echo hello from 42']
).decode('utf-8') == 'hello from 42\n'
mount()
if fixed:
assert not isdir(fdir)
assert not isfile(fname)
return
assert isfile(fname)
assert isdir(fdir)
assert subprocess.check_output(
[join(fdir, '42sh'), '-c', 'echo hello from 42']
).decode('utf-8') == 'hello from 42\n'
rm_rf(fdir)
assert not isdir(fdir)
mount()
assert isfile(fname)
assert not isdir(fdir)
def test_invalid_perm_setting(sub_done):
assert isfile(sub_done, 'dir', 'single_file_work')
with pytest.raises(PermissionError):
chown([sub_done, 'dir', 'single_file_work'], 7, 7)
assert isfile(sub_done, 'dir', 'single_file_work')
with pytest.raises(PermissionError):
chmod([sub_done, 'dir', 'single_file_work'], stat.S_IRGRP)
assert isfile(sub_done, 'dir', 'single_file_work')
def test_delete_files(mount_dir, sub_open, sub_done):
with open(join(sub_open, 'file1'), 'w') as f:
f.write('abc\n')
assert isfile(sub_open, 'file1')
with open(join(sub_open, 'file1'), 'r') as f:
assert f.read() == 'abc\n'
rm(sub_open, 'file1')
assert not isfile(sub_open, 'file1')
with pytest.raises(FileNotFoundError):
rm(sub_open, 'nonexisting')
def test_renaming_file_in_fixed(sub_open, mount):
fname = join(sub_open, 'new_test_file')
fname2 = join(sub_open, 'new_test_file2')
fname3 = join(sub_open, 'new_test_file3')
# Make sure we cannot delete existing files in fixed mode
assert not isfile(fname)
with open(fname, 'w') as f:
f.write('Hello thomas\n')
assert isfile(fname)
mount(fixed=True)
assert isfile(fname)
with pytest.raises(PermissionError):
rename([fname], [fname2])
del fname
assert not isfile(fname2)
with open(fname2, 'w') as f:
f.write('Hello thomas\n')
assert isfile(fname2)
rename([fname2], [fname3])
assert not isfile(fname2)
assert isfile(fname3)
with open(fname3, 'r') as f:
assert f.read() == 'Hello thomas\n'
mount(fixed=True)
assert not isfile(fname3)
def test_editing_file_in_fixed(sub_open, mount):
fname = join(sub_open, 'new_test_file')
# Make sure we cannot delete existing files in fixed mode
assert not isfile(fname)
with open(fname, 'w') as f:
f.write('Hello thomas\n')
assert isfile(fname)
mount(fixed=True)
assert isfile(fname)
with pytest.raises(PermissionError):
with open(fname, 'w') as f:
f.write('hello\n')
def __load(self):
if isfile(self.file):
with open(self.file, 'rb') as fp:
self.data = pickle.load(fp)
if 'items' not in self.data or 'file' not in self.data:
raise FileExistsError(
'Could not resume due to corrupted track file, please remove the file and try again: %s'
% self.file)
if not isfile(self.data['file']):
raise FileNotFoundError(
'Previous output file is missing, unable to resume: %s'
% self.data['file'])
self._oldout = self.data['file']
else:
self.data = {'items': set(), 'file': self._newout}
def probe_cache(cid, fpath='probe_cache/'):
global cached_probes
global fcached_probes
fnum = cid / 1000
fpath = format_dirpath(fpath)
fname = fpath+'pc'+str(fnum)+'.json'
ret = None
scid = str(cid)
if isfile(fname):
with open(fname, 'r+') as f:
pc = json.load(f)
if scid in pc:
ret = pc[scid]
if ret is None:
pc = dict()
try:
tmp = Probe(id=cid)
pc[scid] = {'country': tmp.country_code,
'asn': tmp.asn_v4,
'prefix': tmp.prefix_v4,
'coords': tmp.geometry['coordinates']}
ret = pc[scid]
except Exception as e:
print(e)
print('failed to get '+scid)
if scid in pc:
with open(fname, 'w+') as f:
json.dump(pc, f)
with open(fcached_probes, 'w+') as f:
json.dump(list(cached_probes), f)
cached_probes.add(cid)
if ret is None:
raise Exception('failed to get client info')
else:
return ret
def __add_header(self):
if not isfile(self.file):
if self.firstrow > 0:
for i in range(self.firstrow):
self.append([])
if self.header is not None:
self.append(self.header)
def parse(fpath):
"""
Routes what parser to use depending on the file extension of a file
:param fpath: file path of file
:return: none
"""
# Check existence
if not isfile(fpath):
raise FileNotFoundError('File is not existed')
# Get file extension
ext = getext(fpath)
# Get module
parser = None
for lang, exts in LANGUAGE_EXTENSIONS.items():
if ext in exts:
parser = getattr(sys.modules[__name__], lang)
break
# If extension is not found
if parser is None:
raise FileExistsError('File extension is not supported')
# Parse and return
return parser(fpath)
def test_create_files(mount_dir, sub_open, sub_done):
with pytest.raises(PermissionError):
with open(join(sub_open, 'file1'), 'w') as f:
pass
assert not isfile(sub_open, 'file1')
with pytest.raises(PermissionError):
with open(join(sub_open, 'file1'), 'w') as f:
f.write('abc\n')
assert not isfile(sub_open, 'file1')
with open(join(sub_done, 'file1'), 'w') as f:
pass
assert isfile(sub_done, 'file1')
with open(join(sub_done, 'file2'), 'w') as f:
f.write('abc\n')
assert isfile(sub_done, 'file2')
def test_quote_paths(sub_done, path):
dirs = path.split('/')[:-1]
if len(dirs):
mkdir(join(sub_done, *dirs))
f_path = join(sub_done, path)
with open(join(sub_done, f_path), 'w') as f:
assert f.write('abc') == 3
tmp_path = join(sub_done, 'abc')
rename([f_path], [tmp_path])
assert not isfile(f_path)
assert isfile(tmp_path)
rename([tmp_path], [f_path])
assert isfile(f_path)
assert not isfile(tmp_path)
rm(f_path)
if len(dirs):
rmdir(join(sub_done, *dirs))
def test_bug_fsync(mount_dir, sub_done):
# The `fsync` function didn't return the value of `flush` so editing a
# teacher file didn't update the internal id of the file.
p = join(sub_done, 'dir', 'single_file_work')
assert isfile(p)
with open(p, 'r') as f:
old = f.read()
new = 'NEW CONTENT' + old
fd = os.open(p, os.O_WRONLY | os.O_TRUNC)
os.write(fd, bytes(new, 'utf8'))
os.fsync(fd)
os.close(fd)
with open(p, 'r') as f:
assert new == f.read()
def test_write_to_directory(sub_done):
fdir = join(sub_done, 'new_dir')
fname = join(fdir, 'new_file')
mkdir(fdir)
assert isdir(fdir)
assert not isfile(fdir)
with pytest.raises(IsADirectoryError):
with open(fdir, 'w') as f:
f.write('hello\n')
open(fname, 'w').close()
with pytest.raises(NotADirectoryError):
ls(fname)
with pytest.raises(FileExistsError):
mkdir(fname)
def test_write_and_read_files(mount_dir, sub_open, sub_done):
assert isfile(sub_done, 'dir', 'single_file_work')
with open(join(sub_done, 'dir', 'single_file_work'), 'w') as f:
f.write('abc\n')
assert isfile(sub_done, 'dir', 'single_file_work')
with open(join(sub_done, 'dir', 'single_file_work'), 'r') as f:
assert f.read() == 'abc\n'
with open(join(sub_done, 'file1'), 'a') as f:
f.write('abc\n')
assert isfile(sub_done, 'file1')
with open(join(sub_done, 'file1'), 'r') as f:
assert f.read() == 'abc\n'
with open(join(sub_done, 'file1'), 'w') as f:
f.write('def\n')
with open(join(sub_done, 'file1'), 'r') as f:
assert f.read() == 'def\n'
with open(join(sub_done, 'file1'), 'a') as f:
f.write('def\n')
with open(join(sub_done, 'file1'), 'r') as f:
assert f.read() == 'def\ndef\n'
with pytest.raises(PermissionError):
with open(join(sub_open, 'file1'), 'a') as f:
f.write('def\n')
assert not isfile(sub_open, 'file1')
assert isfile(sub_open, 'dir', 'single_file_work')
with open(join(sub_open, 'dir', 'single_file_work'), 'r') as f:
old = f.read()
with pytest.raises(PermissionError):
with open(join(sub_open, 'dir', 'single_file_work'), 'a') as f:
f.write('abc\n')
assert isfile(sub_open, 'dir', 'single_file_work')
with open(join(sub_open, 'dir', 'single_file_work'), 'r') as f:
assert f.read() == old
def get_probe_distances(data=None, procs=4, domtotal=302, fname0='mapped_probes.pkl'):
print(inspect.stack()[0][3])
if isfile(fname0):
data = pd.read_pickle(fname0)
else:
def get_args():
parser = ArgumentParser(description=APP_DESC, prog=APP_CMD)
parser.add_argument('-f',
'--file',
metavar='FILE',
help='path to the PDF file that is to be parsed')
parser.add_argument(
'-d',
'--dir',
metavar='DIR',
help=
'path to the directory containing the PDF files that are to be parsed')
parser.add_argument('-s',
'--state',
metavar='STATE',
help='state whose PDF rolls(s) are to be parsed')
parser.add_argument('-l',
'--lang',
metavar='LANG',
default='english',
help='specify language used in parsing \
documents (default is English if not being specified)')
parser.add_argument('-o',
'--out',
metavar='FILE',
default=OUTPUT_FILE.format(timestamp(OUTPUT_FILE_TS)),
help='\
specify output file for storing parsed result (must be \'.csv\' file). The default output file is \
\'Parsed-{timestamp}.csv\' and stored in the \'output\' directory')
parser.add_argument('--resume',
action='store_true',
default=False,
help='allows the parsing to be \
resumed later if the program is stopped unexpectedly or intentionally. Only takes effect if it is applied to a directory'
)
parser.add_argument('--version',
action='version',
version=APP_NAME_WITH_VERSION)
parser.add_argument('--all-states',
action='store_true',
default=False,
help='show all the states that are supported and exit')
args = parser.parse_args()
if args.all_states:
out = []
chars = max(len(x) for x in states.keys())
for state, langs in states.items():
out.append('%-{}s : %s'.format(chars) % (state, ', '.join(langs)))
parser.exit(message='States supported:\n%s\n' % '\n'.join(out))
if not args.file and not args.dir or not args.state:
parser.error(
'the following arguments are required: -f/--file or -d/--dir, -s/--state'
)
if args.file and args.dir:
parser.error('only accepts one of --file or --dir at a time')
elif args.file:
if not isfile(args.file):
parser.error('file not found: %s' % args.file)
if not ispdf(args.file):
parser.error('file must be in PDF format')
if args.resume:
parser.error('parsing file does not allow --resume')
elif args.dir:
if not isdir(args.dir):
parser.error('folder not found: %s' % args.dir)
if args.state.lower() not in states:
parser.error('unsupported state \'%s\'' % args.state)
if args.lang and args.lang.lower() not in states[args.state.lower()]:
parser.error('state \'%s\' does not have \'%s\' language' %
(args.state, args.lang))
if dirname(args.out) and not isdir(dirname(args.out)):
parser.error('output folder does not exist: %s' % dirname(args.out))
if filext(args.out).lower() != 'csv':
parser.error('output file must be \'.csv\'')
args.path = args.file or args.dir
args.state = args.state.lower()
args.lang = args.lang.lower()
return args
def cleanup(self):
if isfile(self.file):
rmfile(self.file)
import pyasn
from helpers import format_dirpath, isfile
import json
from ripe.atlas.cousteau import Probe
from IPy import IP
from collections import defaultdict
asndb = pyasn.pyasn('asndb.dat')
fcached_probes = 'cached_probes.list'
import numpy as np
if isfile(fcached_probes):
with open(fcached_probes, 'r+') as f:
cached_probes = set(json.load(f))
else:
cached_probes = set()
def is_public(i):
try:
ip = IP(i)
return ip.iptype() == 'PUBLIC'
except Exception as e:
print(e)
pass
return False
def get_prefix(ip, cid=-1):
prefix = 24
try:
def test_deleting_file_in_fixed(sub_open, mount):
fname = join(sub_open, 'new_test_file')
fname2 = join(sub_open, 'new_test_file2')
# Make sure we cannot delete existing files in fixed mode
assert not isfile(fname)
with open(fname, 'w') as f:
f.write('Hello thomas\n')
assert isfile(fname)
mount(fixed=True)
assert isfile(fname)
with open(fname, 'r') as f:
assert f.read() == 'Hello thomas\n'
with pytest.raises(PermissionError):
with open(fname, 'w') as f:
f.write('Hello thomas\n')
with pytest.raises(PermissionError):
rm(fname)
del fname
# Now make sure we can delete files that did not exist
assert not isfile(fname2)
with open(fname2, 'w') as f:
f.write('Hello thomas2\n')
with open(fname2, 'r') as f:
assert f.read() == 'Hello thomas2\n'
assert isfile(fname2)
rm(fname2)
assert not isfile(fname2)
# Make sure files created in fixed mode are not visible after a remount
assert not isfile(fname2)
with open(fname2, 'w') as f:
f.write('Hello thomas2\n')
with open(fname2, 'r') as f:
assert f.read() == 'Hello thomas2\n'
assert isfile(fname2)
mount(fixed=True)
assert not isfile(fname2)
with open(fname2, 'w') as f:
f.write('Hello thomas2\n')
with open(fname2, 'r') as f:
assert f.read() == 'Hello thomas2\n'
assert isfile(fname2)
mount(fixed=True)
assert not isfile(fname2)
def main():
"""Main program"""
# Check Python
if not minpy(MINIMUM_PYTHON_REQUIRED):
print('Failed! Minimum Python required is %s' %
MINIMUM_PYTHON_REQUIRED)
sys.exit(1)
# Read arguments
try:
args = readargs(**COMMAND_LINE_SETTINGS)
except SyntaxError as msg:
print(msg)
sys.exit(2)
except SystemExit:
sys.exit()
# Load settings
wdir = args.get('dir', DEFAULT_WORKING_DIR)
xml = joinroot(wdir, args.get('xmlfile', DEFAULT_XML_FILE))
out = joinroot(wdir, args.get('outfile', DEFAULT_OUTPUT_FILE))
lang = args.get('lang', None)
anyext = True if 'anyext' in args else False
# Check XML file existence
if not isfile(xml):
print('Failed! XML file is not existed:', xml)
sys.exit(1)
# Check output file existence
if isfile(out):
print('Output file is existed:', out)
confirm = input(
'Do you want to overwrite? (type Y to confirm, any key to exit): ')
if confirm != 'Y' and confirm != 'y':
sys.exit()
# Make sure output file is not a dir
if isdir(out):
print('Failed! Ouput file path is a directory:', out)
sys.exit(1)
# Check output file parent dir
if not dirfound(out, path_is_file=True):
print('Failed! Output file\'s parent dir is not found:', out)
sys.exit(1)
# Do generating
try:
MDGen(infile=xml,
outfile=out,
workdir=wdir,
language=lang,
any_ext=anyext)
except Exception as msg:
print('Failed! Error found: %s' % msg)
sys.exit(2)
# Done
print('MD file successfully generated:', out)
