def _get_pki_test_data_path():
try:
return resource_filename(
Requirement.parse("n6lib"),
'n6lib/tests/certs_and_requests_for_testing/')
finally:
cleanup_resources()
def tearDown(self):
""" Clean Up """
rmtree(self._tmp_base_folder)
# pkg_resources.resources_filename cashes files, when
# in a compiled distribution. We need to clean up potentially
# created files.
pkg_resources.cleanup_resources()
def start():
"""Start client."""
sslcontext = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH)
sslcontext.verify_mode = ssl.CERT_REQUIRED
req = Requirement.parse("all1input")
sslcontext.load_cert_chain(
certfile=resource_filename(
req,
"all1input/{}.crt".format(c.cert_name)),
keyfile=resource_filename(
req,
"all1input/{}.key".format(c.cert_name)))
sslcontext.load_verify_locations(resource_filename(
req,
"all1input/{}.pem".format(c.root_cert_name)))
loop = asyncio.get_event_loop()
try:
while True:
try:
coro = loop.create_connection(
partial(All1InputClientProtocol, loop),
c.ip,
c.port,
ssl=sslcontext)
loop.run_until_complete(coro)
loop.run_forever()
except ConnectionRefusedError:
pass
sleep(5)
except KeyboardInterrupt:
pass
finally:
loop.close()
cleanup_resources()
def tearDown(self):
""" Clean up the sandbox. """
rmtree(self._test_sandbox, ignore_errors=True)
# pkg_resources.resources_filename cashes files, when
# in a compiled distribution. We need to clean up potentially
# created files.
pkg_resources.cleanup_resources()
def _terminate(self, restart=False, app=None):
log.info("Terminating Stoq")
# This removes all temporary files created when calling
# get_resource_filename() that extract files to the file system
import pkg_resources
pkg_resources.cleanup_resources()
log.debug('Stopping deamon')
from stoqlib.lib.daemonutils import stop_daemon
stop_daemon()
# Finally, go out of the reactor and show possible crash reports
yield self._quit_reactor_and_maybe_show_crashreports()
if restart:
from stoqlib.lib.process import Process
log.info('Restarting Stoq')
args = [sys.argv[0], '--no-splash-screen']
if app is not None:
args.append(app)
Process(args)
# os._exit() forces a quit without running atexit handlers
# and does not block on any running threads
# FIXME: This is the wrong solution, we should figure out why there
# are any running threads/processes at this point
log.debug("Terminating by calling os._exit()")
os._exit(0)
raise AssertionError("Should never happen")
def run(param_file, flag_ow=None):
"""
Main function for HYVR generation
Parameters:
param_file (str): Parameter file location
flag_ow : bool
If True, an existing run directory is overwritten.
Returns:
Save data outputs as parameter file
"""
# TODO: implement a seed option to get the same model
# np.random.seed(84)
# Load parameter file
run, model, hydraulics = hp.model_setup(param_file)
hp.set_up_directories(run, param_file, flag_ow)
for sim in range(1, int(run['numsim'])+1):
# Generate facies
model.generate_model()
# model.assign_facies_azim_dip()
assign_facies_azim_dip(model)
# return
if model.generate_hydraulics is True:
# Generate internal heterogeneity
model.generate_hydraulic_parameters(hydraulics)
# from hyvr.postprocess.plotting import cross_section_pcolor
# cross_section_pcolor(model, 'k_iso', log=True, y=10, xlim=[15.5, 16.4], ylim=[7.1, 7.7]
# cross_section_pcolor(model, 'facies', log=False, y=10, xlim=[15.5, 16.4], ylim=[7.1, 7.7])
# cross_section_pcolor(model, 'ha', log=False, y=10, xlim=[15.5, 16.4], ylim=[7.1, 7.7])
# cross_section_pcolor(model, 'hat', log=False, y=10, xlim=[15.5, 16.4], ylim=[7.1, 7.7])
# cross_section_pcolor(model, 'ae', log=False, y=10, xlim=[15.5, 16.4], ylim=[7.1, 7.7])
# cross_section_pcolor(model, 'strata', log=False, y=10, xlim=[15.5, 16.4], ylim=[7.1, 7.7])
# cross_section_pcolor(model, 'dip')
# cross_section_pcolor(model, 'azim')
# Save data
if run['numsim'] > 1:
realname = 'real_{:03d}'.format(sim)
realdir = os.path.join(run['rundir'], realname)
else:
realname = run['runname']
realdir = run['rundir']
hu.try_makefolder(realdir)
create_outputs(model, realdir, realname, run['outputs'])
if param_file == 0:
# this is just the testcase, so we remove the output
from pkg_resources import cleanup_resources
cleanup_resources()
import shutil
runpath = os.path.abspath(run['rundir'])
if os.path.exists(runpath):
shutil.rmtree(runpath)
def start():
"""Start Web server."""
config = OrganiserConfiguration()
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
app = web.Application(debug=config.server.debug)
aiohttp_jinja2.setup(
app,
loader=jinja2.FileSystemLoader(
resource_filename(Requirement.parse('organiser'),
'organiser/templates')),
auto_reload=config.server.debug,
context_processors=[aiohttp_login.flash.context_processor])
aiohttp_session.setup(
app,
EncryptedCookieStorage(config.server.session_secret,
max_age=config.server.session_max_age))
app.middlewares.append(aiohttp_login.flash.middleware)
app['db_client'] = AsyncIOMotorClient()
app['db'] = app['db_client'].organiser
auth_settings = {}
for setting in config.authentication:
auth_settings[setting.upper()] = config.authentication[setting]
aiohttp_login.setup(app, MotorStorage(app['db']), auth_settings)
app.router.add_static("/static/",
path=Path(__file__) / ".." / "static",
name="static")
app.router.add_get("/", index_handler)
app.router.add_get("/calendar/{period}/", calendar_handler)
app.router.add_get("/calendar/{period}/{date}/", calendar_handler)
app.router.add_get("/lists/", list_handler)
web.run_app(app, host=config.server.ip, port=config.server.port)
cleanup_resources()
def tearDown(self): extractor.registered_extractors = [] pkg_resources.cleanup_resources() shutil.rmtree(self.target_path) if self.site.source: self.site.source.stoppedEvent(self.site.source, None) Logger.stop()
def _terminate(self, restart=False, app=None):
log.info("Terminating Stoq")
# This removes all temporary files created when calling
# get_resource_filename() that extract files to the file system
import pkg_resources
pkg_resources.cleanup_resources()
log.debug('Stopping deamon')
from stoqlib.lib.daemonutils import stop_daemon
stop_daemon()
# Finally, go out of the reactor and show possible crash reports
log.debug("Show some crash reports")
self._show_crash_reports()
log.debug("Stoq gtk.main")
import gtk
gtk.main_quit()
if restart:
from stoqlib.lib.process import Process
log.info('Restarting Stoq')
args = [sys.argv[0], '--no-splash-screen']
if app is not None:
args.append(app)
Process(args)
# os._exit() forces a quit without running atexit handlers
# and does not block on any running threads
# FIXME: This is the wrong solution, we should figure out why there
# are any running threads/processes at this point
log.debug("Terminating by calling os._exit()")
os._exit(0)
raise AssertionError("Should never happen")
def read_config() -> dict:
"""
reads the config json file
:return: the contents of the config file
"""
try:
path_components = __name__.split('.')[:-1]
resource_path = '.'.join(path_components)
cached_file = pkg_resources.resource_filename(
resource_path, 'app.prop')
# check if the file exists at the path.
file_contents = json.load(open(cached_file))
# clean up all resources
pkg_resources.cleanup_resources()
# return the file contents
return file_contents
except FileNotFoundError:
print('Could not locate the "app.prop". '
'Please make sure it exists in the same directory.')
except json.JSONDecodeError:
print("The configuration file must have a valid JSON.")
except TypeError:
print("The configuration file must have a valid JSON.")
def tearDownClass(cls):
"""
Removes test temp directory and deletes all files
:return:
"""
pkg_resources.cleanup_resources()
shutil.rmtree(cls.test_dir) # remove temp directory and delete all files
def tearDown(self):
"""Clean up after each test."""
pkg_resources.cleanup_resources()
assert(self.tmpfhandle is None)
if self.tmpfname is not None:
os.unlink(self.tmpfname)
self.tmpfname = None
def tearDownClass(cls):
pkg_resources.cleanup_resources()
for d in [
cls.prediction_directory, cls.prediction_directory_2,
cls.prediction_directory_3, cls.groundtruth_directory,
cls.groundtruth_2_directory
]:
shutil.rmtree(d)
def get_config_base_lines():
try:
filename = resource_filename(Requirement.parse('n6sdk'),
'n6sdk/_api_test_tool/config_base.ini')
with open(filename) as f:
return f.read().splitlines()
finally:
cleanup_resources()
def get_config_base_lines():
try:
filename = resource_filename(Requirement.parse('n6sdk'),
'n6sdk/_api_test_tool/config_base.ini')
with open(filename) as f:
return f.read().splitlines()
finally:
cleanup_resources()
def iter_config_base_lines():
try:
filename = resource_filename(Requirement.parse('n6sdk-py2'),
'n6sdk/_api_test_tool/config_base.ini')
with open(filename, 'rb') as f:
for line in f.read().splitlines():
yield line.decode('utf-8')
finally:
cleanup_resources()
def main():
"""main."""
demo_path = 'funkload-demo'
print "Extract FunkLoad examples into ./%s : ... " % demo_path,
cache_path = resource_filename('funkload', 'demo')
demo_path = os.path.join(os.path.abspath(os.path.curdir), demo_path)
copytree(cache_path, demo_path)
cleanup_resources()
print "done."
def execute_freesasa_api(structure):
"""
Calls freesasa using its Python API and returns
per-residue accessibilities.
"""
try:
from freesasa import Classifier, structureFromBioPDB, calc
except ImportError as err:
print(
'[!] The binding affinity prediction tools require the \'freesasa\' Python API',
file=sys.stderr)
raise ImportError(err)
asa_data, rsa_data = {}, {}
_rsa = rel_asa['total']
config_path = os.environ.get(
'FREESASA_PAR',
pkg_resources.resource_filename('prodigy', 'naccess.config'))
classifier = Classifier(config_path)
pkg_resources.cleanup_resources()
# classifier = freesasa.Classifier( os.environ["FREESASA_PAR"])
# Disable
with stdchannel_redirected(sys.stderr, os.devnull):
try:
struct = structureFromBioPDB(
structure,
classifier,
)
result = calc(struct)
except AssertionError as e:
error_message = '\n[!] Error when running freesasa: \n[!] {}'.format(
e)
print(error_message)
raise Exception(error_message)
# iterate over all atoms to get SASA and residue name
for idx in range(struct.nAtoms()):
atname = struct.atomName(idx)
resname = struct.residueName(idx)
resid = struct.residueNumber(idx)
chain = struct.chainLabel(idx)
at_uid = (chain, resname, resid, atname)
res_uid = (chain, resname, resid)
asa = result.atomArea(idx)
asa_data[at_uid] = asa
# add asa to residue
rsa_data[res_uid] = rsa_data.get(res_uid, 0) + asa
# convert total asa ro relative asa
rsa_data.update(
(res_uid, asa / _rsa[res_uid[1]]) for res_uid, asa in rsa_data.items())
return asa_data, rsa_data
def start():
"""Start server."""
global loop, ignore_devices, all_devices, STOP, devices
req = Requirement.parse("all1input")
server_ssl = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
server_ssl.verify_mode = ssl.CERT_REQUIRED
server_ssl.load_cert_chain(
certfile=resource_filename(
req,
"all1input/{}.crt".format(c.server_cert_name)),
keyfile=resource_filename(
req,
"all1input/{}.key".format(c.server_cert_name)))
server_ssl.load_verify_locations(resource_filename(
req,
"all1input/{}.pem".format(c.root_cert_name)))
client_ssl = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH)
client_ssl.verify_mode = ssl.CERT_REQUIRED
client_ssl.load_cert_chain(
certfile=resource_filename(
req,
"all1input/{}.crt".format(c.cert_name)),
keyfile=resource_filename(
req,
"all1input/{}.key".format(c.cert_name)))
client_ssl.load_verify_locations(resource_filename(
req,
"all1input/{}.pem".format(c.root_cert_name)))
all_devices = [evdev.InputDevice(fn) for fn in evdev.list_devices()]
devices = {}
ignore_devices = []
loop = asyncio.get_event_loop()
coro_server = loop.create_server(
All1InputServerClientProtocol, c.ip, c.port, ssl=server_ssl)
server = loop.run_until_complete(coro_server)
coro_client = loop.create_connection(
partial(All1InputClientProtocol, loop),
c.ip,
c.port,
ssl=client_ssl)
loop.run_until_complete(coro_client)
loop.call_later(1, update_devices)
loop.call_later(0.01, move_mouse)
try:
loop.run_forever()
except KeyboardInterrupt:
STOP = True
finally:
for dev in devices:
devices[dev].ungrab()
devices[dev].close()
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
cleanup_resources()
def load_prefs():
prefs_file = os.path.join(os.path.expanduser('~'), '.dapplerc')
if not os.path.exists(prefs_file):
shutil.copy(resource_filename(__name__, 'defaults/_dapplerc'),
prefs_file)
cleanup_resources()
with open(prefs_file, 'r') as f:
return yaml.load(f)
def load_prefs():
prefs_file = os.path.join(os.path.expanduser('~'), '.dapplerc')
if not os.path.exists(prefs_file):
shutil.copy(resource_filename(__name__, 'defaults/_dapplerc'),
prefs_file)
cleanup_resources()
with open(prefs_file, 'r') as f:
return yaml.load(f)
async def _exec_on_midi_end(proc: Process, func: Callable, port: str) -> int:
"""Exec func when midi stops playing."""
return_code = await proc.wait()
if proc.stdout is None:
reset_proc = await create_subprocess_exec(
'pmidi', '-p', port, '-d', '0',
resource_filename(Requirement.parse("voicetrainer"),
'voicetrainer/reset.midi'))
await reset_proc.wait()
cleanup_resources()
_PROC_LOCK.release()
await func()
return return_code
def execute(self):
""" Run the scanner """
try:
self._start_zap()
if not self._wait_for_zap_start():
log.error("ZAP failed to start")
error = Error(
tool=self.get_name(),
error="ZAP failed to start",
details="ZAP daemon failed to start"
)
self.errors.append(error)
return
log.info("Target: %s", self.config.get("target"))
self._prepare_context()
self._setup_scan_policy()
self._spider()
self._wait_for_passive_scan()
self._ajax_spider()
self._wait_for_passive_scan()
self._active_scan()
self._wait_for_passive_scan()
except:
log.exception("Exception during ZAP scanning")
error = Error(
tool=self.get_name(),
error=f"Exception during ZAP scanning",
details=f"```\n{traceback.format_exc()}\n```"
)
self.errors.append(error)
finally:
try:
# Get report
log.info("Getting ZAP report")
zap_report = self._zap_api.core.jsonreport()
# Parse JSON
log.info("Processing findings")
parse_findings(zap_report, self)
except:
log.exception("Exception during ZAP findings processing")
error = Error(
tool=self.get_name(),
error=f"Exception during ZAP findings processing",
details=f"```\n{traceback.format_exc()}\n```"
)
self.errors.append(error)
self._save_intermediates()
pkg_resources.cleanup_resources()
self._stop_zap()
def init():
shutil.copytree(resource_filename(__name__, 'defaults/_dapple'), 'dapple')
# resource_filename leaves behind temp files.
# cleanup_resources deletes them.
cleanup_resources()
try:
load_plugins()
(dapple.plugins.load('core.install_package'))(
'dapple', ipfs='QmU9tDS5UebJ3mhakB2GtiJ3E2zbn2b1xaXzUNkig9Qw41')
except:
print("ERROR: Could not pull `dapple` package from IPFS! "
"You might try installing it manually.", file=sys.stderr)
print("Init'ed Dapple package. You might want to edit dapple/dappfile now.")
def make_profile():
"""
Create a local profile in user's home folder.
"""
home_path = os.path.expanduser('~/.ScatPy/')
try:
os.makedirs(home_path)
except OSError:
raise(IOError('A folder ~./ScatPy already exists. Delete it manually before proceeding.'))
files = resource_listdir('ScatPy', 'profiles')
for f in files:
src = resource_filename('ScatPy', os.path.join('profiles', f))
shutil.copy(src, home_path)
cleanup_resources()
def init():
shutil.copytree(resource_filename(__name__, 'defaults/_dapple'), '.dapple')
# resource_filename leaves behind temp files.
# cleanup_resources deletes them.
cleanup_resources()
try:
load_plugins()
(dapple.plugins.load('core.install_package'))(
'core', ipfs='QmP3sVwo7nRxekM37pyLL4AS7E1rX5qnAnFNGLpgwgZsHj')
except:
print("ERROR: Could not pull `core` package from IPFS! "
"You might try installing it manually.", file=sys.stderr)
print("Init'ed Dapple package. You might want to edit .dapple/dappfile now.")
def make_profile():
"""
Create a local profile in user's home folder.
"""
home_path = os.path.expanduser('~/.ScatPy/')
try:
os.makedirs(home_path)
except OSError:
raise IOError
files = resource_listdir('ScatPy', 'profiles')
for f in files:
src = resource_filename('ScatPy', os.path.join('profiles', f))
shutil.copy(src, home_path)
cleanup_resources()
def predict(self, document):
"""
tag the new data basde on a given model
:param document: document object
:returns: list of predictions
"""
logger.info('Start to predict')
y,x = self.fit(document)
if Svm._auto_config:
file_name = self._auto_load('file', self.model_file)
m = svmutil.svm_load_model(file_name)
pkg_resources.cleanup_resources()
else :
m = svmutil.svm_load_model(self.model_file)
y_pred, p_acc, p_val = svmutil.svm_predict(y, x, m, "-q")
return y_pred
def init():
shutil.copytree(resource_filename(__name__, 'defaults/_dapple'), 'dapple')
# resource_filename leaves behind temp files.
# cleanup_resources deletes them.
cleanup_resources()
try:
load_plugins()
(dapple.plugins.load('core.install_package'))(
'dapple', ipfs='QmU9tDS5UebJ3mhakB2GtiJ3E2zbn2b1xaXzUNkig9Qw41')
except:
print(
"ERROR: Could not pull `dapple` package from IPFS! "
"You might try installing it manually.",
file=sys.stderr)
print(
"Init'ed Dapple package. You might want to edit dapple/dappfile now.")
def make_profile():
"""
Create a local profile in user's home folder.
"""
home_path = os.path.expanduser('~/.ScatPy/')
try:
os.makedirs(home_path)
except OSError:
raise (IOError(
'A folder ~./ScatPy already exists. Delete it manually before proceeding.'
))
files = resource_listdir('ScatPy', 'profiles')
for f in files:
src = resource_filename('ScatPy', os.path.join('profiles', f))
shutil.copy(src, home_path)
cleanup_resources()
def _terminate(self, restart=False, app=None):
log.info("Terminating Stoq")
# This removes all temporary files created when calling
# get_resource_filename() that extract files to the file system
import pkg_resources
pkg_resources.cleanup_resources()
log.debug('Stopping deamon')
from stoqlib.lib.daemonutils import stop_daemon
stop_daemon()
# Finally, go out of the reactor and show possible crash reports
log.debug("Show some crash reports")
self._show_crash_reports()
# Make sure that no connection is left open (specially on Windows)
try:
from stoqlib.database.runtime import get_default_store
get_default_store().close()
except Exception:
pass
if restart:
from stoqlib.lib.process import Process
log.info('Restarting Stoq')
args = [sys.argv[0], '--no-splash-screen']
if app is not None:
args.append(app)
Process(args)
# os._exit() forces a quit without running atexit handlers
# and does not block on any running threads
# FIXME: This is the wrong solution, we should figure out why there
# are any running threads/processes at this point
log.debug("Terminating by calling os._exit()")
os._exit(0)
raise AssertionError("Should never happen")
def start():
"""Start Web server."""
config = DndConfiguration()
# asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
app = web.Application(debug=config.server.debug)
aiohttp_jinja2.setup(
app,
loader=jinja2.FileSystemLoader(
resource_filename(Requirement.parse('dnd'), 'dnd/templates')),
auto_reload=config.server.debug,
context_processors=[aiohttp_login.flash.context_processor])
aiohttp_session.setup(
app,
EncryptedCookieStorage(config.server.session_secret,
max_age=config.server.session_max_age))
aiohttp_jinja2.get_env(app).filters['to_roman'] = toRoman
aiohttp_jinja2.get_env(app).filters['max'] = max
aiohttp_jinja2.get_env(app).filters['cutoff_dict'] = _cutoff_dict_filter
app.middlewares.append(aiohttp_login.flash.middleware)
app['db_client'] = AsyncIOMotorClient()
app['db'] = app['db_client'].dnd
auth_settings = {}
for setting in config.authentication:
auth_settings[setting.upper()] = config.authentication[setting]
aiohttp_login.setup(app, MotorStorage(app['db']), auth_settings)
app.router.add_static("/static/",
path=Path(__file__) / ".." / "static",
name="static")
app.router.add_get("/", index_handler)
app.router.add_post("/api/new-character/", new_character_data_handler)
app.router.add_get("/{id}/{name}/", character_handler)
app.router.add_post("/api/{id}/{attribute}/{extra}/", data_handler)
app.router.add_post("/api/{id}/{attribute}/", data_handler)
web.run_app(app, host=config.server.ip, port=config.server.port)
cleanup_resources()
def execute(self):
""" Run the scanner """
# Get config
config_data = pkg_resources.resource_string(
"dusty",
f"{'/'.join(__name__.split('.')[1:-1])}/data/w3af_full_audit.w3af"
)
if self.config.get("config_file", None):
with open(self.config.get("config_file"), "r") as config:
config_data = config.read()
# Make temporary files
config_file_fd, config_file = tempfile.mkstemp()
output_file_fd, output_file = tempfile.mkstemp()
log.debug("Config file: %s", config_file)
log.debug("Output file: %s", output_file)
# Fill config data variables
config_data = config_data.decode("utf-8").format(
target=self.config.get("target"),
output_section=constants.W3AF_OUTPUT_SECTION.replace("{output_file}", output_file)
)
os.write(config_file_fd, config_data.encode("utf-8"))
# Close unneeded handles
os.close(config_file_fd)
os.close(output_file_fd)
# Run scanner
task = subprocess.run([
"w3af_console", "-y", "-n", "-s", config_file
], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
log.log_subprocess_result(task)
# Parse findings
parse_findings(output_file, self)
# Save intermediates
self.save_intermediates(output_file, config_file, task)
# Remove temporary files
os.remove(config_file)
os.remove(output_file)
pkg_resources.cleanup_resources()
def main():
args = parse_arguments()
try:
with tempfile.TemporaryDirectory() as temp_dir:
backend = os.path.join(get_resource_dir('drawio/war'),
'export2.html')
frontend = os.path.join(temp_dir, 'frontend.js')
with open(frontend, 'wb') as frontend_file:
in_data = get_resource('frontend.js.in')
in_data = in_data.replace(
b'@FILE@', bytearray("'" + backend + "'", 'ASCII'))
frontend_file.write(in_data)
# chop newlines from input
processed_input = os.path.join(temp_dir, 'input')
with open(processed_input, 'w') as in_file:
data = args.input.readlines()
data = [l.strip() for l in data]
in_file.write(''.join(data))
with open(processed_input, 'r') as in_file:
command = [args.phantomjs.name,
frontend,
args.format,
str(args.scale),
str(args.quality)]
subprocess.check_call(
command,
stdout=args.output,
stdin=in_file)
args.output.close()
finally:
pkg_resources.cleanup_resources()
def start():
"""Start cert manager."""
parser = argparse.ArgumentParser(
prog="cert_manager", description='Create certificates for all1input.')
parser.add_argument(
'action',
choices=["all", "client"],
help=("all: create root, server and client cert from config |"
"client: creates client certificate with NAME"))
parser.add_argument("--name",
default="client",
help="certificate name to use for client certificate")
args = parser.parse_args()
if args.action == "all":
try:
print("Making root CA")
create_root_ca(c.root_cert_name)
except FileExists:
print("Root CA already exists")
finally:
cleanup_resources()
try:
print("Making server certificate")
create_certificate(c.server_cert_name, True, c.root_cert_name,
c.ip)
except FileExists:
print("Server cert already exists")
finally:
cleanup_resources()
try:
print("Making client certificate")
create_certificate(c.cert_name, False, c.root_cert_name)
except FileExists:
print("Client cert already exists")
finally:
cleanup_resources()
elif args.action == "client":
try:
print("Making client certificate")
create_certificate(args.name, False, c.root_cert_name)
finally:
cleanup_resources()
def tearDownModule():
"""Remove test dataset files after unit tests are complete"""
if test_info["temp_file_path"] != "":
shutil.rmtree(test_info["temp_file_path"])
cleanup_resources()
def tearDownClass(cls):
"""Clean up temp files after tests are complete"""
cleanup_resources()
shutil.rmtree(cls.optdict["outdir"])
def stats(self, resultset=()):
slices = OrderedDict()
datadict = defaultdict(list)
# 1: Create a giant RDF graph consisting of all triples of all
# repos' commondata. To avoid parsing the same RDF files
# over and over, this section duplicates the logic of
# DocumentRepository.commondata to make sure each RDF
# file is loaded only once.
ttlfiles = set()
resource_graph = Graph()
namespaces = {}
for repo in self.repos:
for prefix, ns in repo.make_graph().namespaces():
assert ns not in namespaces or namespaces[
ns] == prefix, "Conflicting prefixes for ns %s" % ns
namespaces[ns] = prefix
resource_graph.bind(prefix, ns)
for cls in inspect.getmro(repo.__class__):
if hasattr(cls, "alias"):
commonpath = "res/extra/%s.ttl" % cls.alias
if os.path.exists(commonpath):
ttlfiles.add(commonpath)
elif pkg_resources.resource_exists(
'ferenda', commonpath):
ttlfiles.add(
pkg_resources.resource_filename(
'ferenda', commonpath))
self.log.debug(
"stats: Loading resources %s into a common resource graph" %
list(ttlfiles))
for filename in ttlfiles:
resource_graph.parse(data=util.readfile(filename), format="turtle")
pkg_resources.cleanup_resources()
# 2: if used in the resultset mode, only calculate stats for those
# resources/documents that are in the resultset.
resultsetmembers = set()
if resultset:
for r in resultset:
resultsetmembers.add(r['iri'])
# 3: using each repo's faceted_data and its defined facet
# selectors, create a set of observations for that repo
#
# FIXME: If in resultset mode, we might ask a repo for its
# faceted data and then use exactly none of it since it
# doesn't match anything in resultsetmembers. We COULD analyze
# common resultset iri prefixes and then only call
# faceted_data for some (or one) repo.
for repo in self.repos:
data = repo.faceted_data()
if resultsetmembers:
data = [r for r in data if r['uri'] in resultsetmembers]
for facet in repo.facets():
if not facet.dimension_type:
continue
dimension, obs = self.stats_slice(data, facet, resource_graph)
if dimension in slices:
# since observations is a Counter not a regular
# dict, if slices[dimensions] and observations
# have common keys this will add the counts not
# replace them.
slices[dimension].update(obs)
else:
slices[dimension] = obs
# 4. Transform our easily-updated data structures to the list
# of dicts of lists that we're supposed to return.
res = {"type": "DataSet", "slices": []}
for k, v in sorted(slices.items()):
observations = []
for ok, ov in sorted(v.items()):
observations.append({ok[0]: ok[1], "count": ov})
res['slices'].append({
"dimension": k,
"observations": observations
})
return res
log_exception("Shutdown failed.", Logtype.ERROR)
finally:
seconds_left = 30
while seconds_left > 0 and threading.active_count() > 2: #main + logging thread
time.sleep(1)
seconds_left -= 1
if threading.active_count() > 2: #main + logging thread
print("Hanging threads:")
for t in threading.enumerate():
if not(t == threading.current_thread()) and t.isAlive():
print("* " + t.name)
print("Force quit.")
Logger.stop()
sys.exit(0) #TODO: does this really kill all threads?
Logger.stop()
pkg_resources.cleanup_resources(force=False)
def signal_handler(signal, frame):
log("Shutdown requested by user.", Logtype.INFO)
shutdown()
def log(msg, logtype = Logtype.NONE):
Logger.log("witica", msg, logtype)
def log_exception(msg, logtype = Logtype.NONE, exc_info=None):
Logger.log_exception("witica", msg, logtype, exc_info)
def create_source(args):
if args.source:
return Source.construct_from_file(args.source)
else:
def tearDownModule():
"""Remove test dataset files after unit tests are complete"""
cleanup_resources()
def tearDown(self): extractor.registered_extractors = [] pkg_resources.cleanup_resources() Logger.stop()
def tearDownModule():
cleanup_resources()
def close(self):
self.db.close()
super().close()
pkg_resources.cleanup_resources()
def stats(self, resultset=()):
slices = OrderedDict()
datadict = defaultdict(list)
# 1: Create a giant RDF graph consisting of all triples of all
# repos' commondata. To avoid parsing the same RDF files
# over and over, this section duplicates the logic of
# DocumentRepository.commondata to make sure each RDF
# file is loaded only once.
ttlfiles = set()
resource_graph = Graph()
namespaces = {}
for repo in self.repos:
for prefix, ns in repo.make_graph().namespaces():
assert ns not in namespaces or namespaces[ns] == prefix, "Conflicting prefixes for ns %s" % ns
namespaces[ns] = prefix
resource_graph.bind(prefix, ns)
for cls in inspect.getmro(repo.__class__):
if hasattr(cls, "alias"):
commonpath = "res/extra/%s.ttl" % cls.alias
if os.path.exists(commonpath):
ttlfiles.add(commonpath)
elif pkg_resources.resource_exists('ferenda', commonpath):
ttlfiles.add(pkg_resources.resource_filename('ferenda', commonpath))
self.log.debug("stats: Loading resources %s into a common resource graph" %
list(ttlfiles))
for filename in ttlfiles:
resource_graph.parse(data=util.readfile(filename), format="turtle")
pkg_resources.cleanup_resources()
# 2: if used in the resultset mode, only calculate stats for those
# resources/documents that are in the resultset.
resultsetmembers = set()
if resultset:
for r in resultset:
resultsetmembers.add(r['iri'])
# 3: using each repo's faceted_data and its defined facet
# selectors, create a set of observations for that repo
#
# FIXME: If in resultset mode, we might ask a repo for its
# faceted data and then use exactly none of it since it
# doesn't match anything in resultsetmembers. We COULD analyze
# common resultset iri prefixes and then only call
# faceted_data for some (or one) repo.
for repo in self.repos:
data = repo.faceted_data()
if resultsetmembers:
data = [r for r in data if r['uri'] in resultsetmembers]
for facet in repo.facets():
if not facet.dimension_type:
continue
dimension, obs = self.stats_slice(data, facet, resource_graph)
if dimension in slices:
# since observations is a Counter not a regular
# dict, if slices[dimensions] and observations
# have common keys this will add the counts not
# replace them.
slices[dimension].update(obs)
else:
slices[dimension] = obs
# 4. Transform our easily-updated data structures to the list
# of dicts of lists that we're supposed to return.
res = {"type": "DataSet",
"slices": []
}
for k, v in sorted(slices.items()):
observations = []
for ok, ov in sorted(v.items()):
observations.append({ok[0]: ok[1],
"count": ov})
res['slices'].append({"dimension": k,
"observations": observations})
return res
def main():
ap = argparse.ArgumentParser()
ap.add_argument('source', type=str,
choices=['twitter', 'folder', 'txtfile', 'stdin', 'demo'],
help='Where the input data should come from.')
ap.add_argument('query', type=str, nargs=argparse.REMAINDER,
help='Query for Twitter search.')
ap.add_argument('-p', '--percent', type=float, default=0.25,
help="Percentage of clusters to return as main topics.")
ap.add_argument('-n', '--number', type=int, default=2,
help="Number of representative sentences from each topic.")
ap.add_argument('-t', '--no-tokenize', action='store_true',
help="When present, don't tokenize input string.")
args = ap.parse_args()
print("reading source text...", "({})".format(args.source))
if args.source == 'twitter':
print("with query:", args.query)
input_text = search_twitter(args.query)
elif args.source == 'folder':
input_text = ""
path = args.query[0]
for resfile in next(os.walk(path))[2]:
input_text += open(path + '/' + resfile,
encoding='utf-8').read()
elif args.source == 'txtfile':
input_text = open(args.query[0], "r", encoding="utf-8").read()
elif args.source == 'stdin':
input_text = sys.stdin.read()
elif args.source == 'demo':
input_text = demo()
else:
print(args.usage)
sys.exit(1)
print("tokenizing...")
if not args.no_tokenize: tokenized_input = pre.tokenizes(input_text)
print("tagging...")
if not args.no_tokenize:
tagged_input = pre.pos_tag(tokenized_input)
else:
tagged_input = pre.pos_tag(input_text.encode())
print("generating sentence objects... ", end="")
sentences = pre.make_sent_objs(tagged_input)
print("({} sentences)".format(len(sentences)))
print("computing matrix for clustering...")
X = clst.build_sent_matrix(sentences, weighting='binary')
print("clustering...")
clusters, eps = clst.get_clusters(X, sentences,
min_samp=int(len(sentences)*0.04))
print("\u03B5 set to", eps)
print(clst.cluster_sizes(clusters))
print("selecting top sentences...")
num_to_take = int(args.percent * len(clusters))
if num_to_take < 2:
num_to_take = len(clusters)
toptop = sel.top_n_topics(num_to_take, clusters, len(sentences))
# output result
print("result:")
print_toptopics(toptop, sentences, clusters, args.number)
cleanup_resources()
exit(0)
def tearDown(self):
pkg_resources.cleanup_resources()
def tearDownClass(cls):
pkg_resources.cleanup_resources()
shutil.rmtree(cls.prediction_directory)
def zap(config):
# Nested functions
def _wait_for_completion(condition, status, message, interval=10):
""" Watch progress """
current_status = status()
logging.info(message, current_status)
while condition():
sleep(interval)
next_status = status()
if next_status != current_status:
logging.info(message, next_status)
current_status = next_status
# ZAP wrapper
tool_name = "ZAP"
results = list()
# Start ZAP daemon in background (no need for supervisord)
logging.info("Starting ZAP daemon")
zap_daemon = subprocess.Popen([
"/usr/bin/java", "-Xmx499m", "-jar", "/opt/zap/zap.jar", "-daemon",
"-port", "8091", "-host", "0.0.0.0", "-config", "api.key=dusty",
"-config", "api.addrs.addr.regex=true", "-config",
"api.addrs.addr.name=.*", "-config",
"ajaxSpider.browserId=htmlunit"
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
zap_api = ZAPv2(apikey="dusty",
proxies={
"http": "http://127.0.0.1:8091",
"https": "http://127.0.0.1:8091"
})
# Wait for zap to start
zap_started = False
for _ in range(600):
try:
logging.info("Started ZAP %s", zap_api.core.version)
zap_started = True
break
except IOError:
sleep(1)
if not zap_started:
logging.error("ZAP failed to start")
zap_daemon.kill()
zap_daemon.wait()
return tool_name, results
# Format target URL
proto = config.get("protocol")
host = config.get("host")
port = config.get("port")
target = f"{proto}://{host}"
if (proto == "http" and int(port) != 80) or \
(proto == "https" and int(port) != 443):
target = f"{target}:{port}"
logging.info("Scanning target %s", target)
# Setup context
logging.info("Preparing context")
zap_context_name = "dusty"
zap_context = zap_api.context.new_context(zap_context_name)
# Setup context inclusions and exclusions
zap_api.context.include_in_context(zap_context_name,
f".*{re.escape(host)}.*")
for include_regex in config.get("include", list()):
zap_api.context.include_in_context(zap_context_name, include_regex)
for exclude_regex in config.get("exclude", list()):
zap_api.context.exclude_from_context(zap_context_name,
exclude_regex)
if config.get("auth_script", None):
# Load our authentication script
zap_api.script.load(scriptname="zap-selenium-login.js",
scripttype="authentication",
scriptengine="Oracle Nashorn",
filename=pkg_resources.resource_filename(
"dusty",
"templates/zap-selenium-login.js"),
scriptdescription="Login via selenium script")
# Enable use of laoded script with supplied selenium-like script
zap_api.authentication.set_authentication_method(
zap_context, "scriptBasedAuthentication",
urllib.parse.urlencode({
"scriptName":
"zap-selenium-login.js",
"Script":
base64.b64encode(
json.dumps(config.get("auth_script")).encode(
"utf-8")).decode("utf-8")
}))
# Add user to context
zap_user = zap_api.users.new_user(zap_context, "dusty_user")
zap_api.users.set_authentication_credentials(
zap_context, zap_user,
urllib.parse.urlencode({
"Username":
config.get("auth_login", ""),
"Password":
config.get("auth_password", ""),
"type":
"UsernamePasswordAuthenticationCredentials"
}))
# Enable added user
zap_api.users.set_user_enabled(zap_context, zap_user, True)
# Setup auth indicators
if config.get("logged_in_indicator", None):
zap_api.authentication.set_logged_in_indicator(
zap_context, config.get("logged_in_indicator"))
if config.get("logged_out_indicator", None):
zap_api.authentication.set_logged_out_indicator(
zap_context, config.get("logged_out_indicator"))
# Setup scan policy
scan_policy_name = "Default Policy"
scan_policies = [
item.strip() for item in config.get("scan_types", "all").split(",")
]
# Disable globally blacklisted rules
for item in c.ZAP_BLACKLISTED_RULES:
zap_api.ascan.set_scanner_alert_threshold(
id=item, alertthreshold="OFF", scanpolicyname=scan_policy_name)
zap_api.pscan.set_scanner_alert_threshold(id=item,
alertthreshold="OFF")
if "all" not in scan_policies:
# Disable all scanners first
for item in zap_api.ascan.scanners(scan_policy_name):
zap_api.ascan.set_scanner_alert_threshold(
id=item["id"],
alertthreshold="OFF",
scanpolicyname=scan_policy_name)
# Enable scanners from suite
for policy in scan_policies:
for item in c.ZAP_SCAN_POCILICES.get(policy, []):
zap_api.ascan.set_scanner_alert_threshold(
id=item,
alertthreshold="DEFAULT",
scanpolicyname=scan_policy_name)
# Spider
logging.info("Spidering target: %s", target)
if config.get("auth_script", None):
scan_id = zap_api.spider.scan_as_user(zap_context,
zap_user,
target,
recurse=True,
subtreeonly=True)
else:
scan_id = zap_api.spider.scan(target)
_wait_for_completion(lambda: int(zap_api.spider.status(scan_id)) < 100,
lambda: int(zap_api.spider.status(scan_id)),
"Spidering progress: %d%%")
# Wait for passive scan
_wait_for_completion(lambda: int(zap_api.pscan.records_to_scan) > 0,
lambda: int(zap_api.pscan.records_to_scan),
"Passive scan queue: %d items")
# Ajax Spider
logging.info("Ajax spidering target: %s", target)
if config.get("auth_script", None):
scan_id = zap_api.ajaxSpider.scan_as_user(zap_context_name,
"dusty_user",
target,
subtreeonly=True)
else:
scan_id = zap_api.ajaxSpider.scan(target)
_wait_for_completion(lambda: zap_api.ajaxSpider.status == 'running',
lambda: int(zap_api.ajaxSpider.number_of_results),
"Ajax spider found: %d URLs")
# Wait for passive scan
_wait_for_completion(lambda: int(zap_api.pscan.records_to_scan) > 0,
lambda: int(zap_api.pscan.records_to_scan),
"Passive scan queue: %d items")
# Active scan
logging.info("Active scan against target %s", target)
if config.get("auth_script", None):
scan_id = zap_api.ascan.scan_as_user(
target,
zap_context,
zap_user,
recurse=True,
scanpolicyname=scan_policy_name)
else:
scan_id = zap_api.ascan.scan(target,
scanpolicyname=scan_policy_name)
_wait_for_completion(lambda: int(zap_api.ascan.status(scan_id)) < 100,
lambda: int(zap_api.ascan.status(scan_id)),
"Active scan progress: %d%%")
# Wait for passive scan
_wait_for_completion(lambda: int(zap_api.pscan.records_to_scan) > 0,
lambda: int(zap_api.pscan.records_to_scan),
"Passive scan queue: %d items")
# Get report
logging.info("Scan finished. Processing results")
zap_report = zap_api.core.jsonreport()
if os.environ.get("debug", False):
with open("/tmp/zap.json", "wb") as report_file:
report_file.write(zap_report.encode("utf-8"))
# Stop zap
zap_daemon.kill()
zap_daemon.wait()
# Parse JSON
results.extend(ZapJsonParser(zap_report, tool_name).items)
pkg_resources.cleanup_resources()
return tool_name, results
def tearDown(self):
"""Clean up after each test."""
pkg_resources.cleanup_resources()
def test_exit_status():
# define columns
cols = {
"intA" : numpy.random.randint(0,high=2**16,size=size),
"intB" : numpy.random.randint(-10,high=20,size=size),
"idxA" : numpy.arange(size),
"chrA" : numpy.array([chr(65+(X%(91-65))) for X in range(size)]),
"strA" : numpy.array([str(GenomicSegment("chrA",X,X+500,"+")) for X in range(size)]),
"strB" : numpy.array([str(GenomicSegment("chrB",X/2,X/2+500,"-")) for X in range(size)]),
"floatA" : 10*numpy.random.randn(size) + 500,
"floatB" : (10**-5)*numpy.random.random(size),
"objA" : numpy.tile(None,5000),
"objB" : numpy.array([GenomicSegment("chrC",X,X+Y,"+") for X,Y in zip(range(size),numpy.random.randint(2,high=1000,size=size))]),
}
# allocate temp files we will use
headerfile = NamedTemporaryFile(delete=False,mode="w")
headerfile_extra_cols = NamedTemporaryFile(delete=False,mode="w")
headerfile_extra_cols_diff = NamedTemporaryFile(delete=False,mode="w")
headerfile_extra_cols_shuffled = NamedTemporaryFile(delete=False,mode="w")
headerfile_shuffled = NamedTemporaryFile(delete=False,mode="w")
headerfile_diff_vals = NamedTemporaryFile(delete=False,mode="w")
noheaderfile = NamedTemporaryFile(delete=False,mode="w")
noheaderfile_extra_cols = NamedTemporaryFile(delete=False,mode="w")
noheaderfile_extra_cols_diff = NamedTemporaryFile(delete=False,mode="w")
noheaderfile_extra_cols_shuffled = NamedTemporaryFile(delete=False,mode="w")
noheaderfile_shuffled = NamedTemporaryFile(delete=False,mode="w")
noheaderfile_diff_vals = NamedTemporaryFile(delete=False,mode="w")
# write values
keyorder = ["idxA"] + sorted(list(set(cols.keys()) - { "idxA" }))
table1 = pd.DataFrame(cols)
table1.to_csv(headerfile,index=False,header=True,sep="\t")
table1.to_csv(noheaderfile,index=False,header=False,sep="\t",
columns=keyorder)
headerfile.close()
noheaderfile.close()
table1["extra"] = 2**7 * numpy.random.random(size=size)
table1.to_csv(headerfile_extra_cols,index=False,header=True,sep="\t")
table1.to_csv(noheaderfile_extra_cols,index=False,header=False,sep="\t",
columns=["extra"]+keyorder)
headerfile_extra_cols.close()
noheaderfile_extra_cols.close()
table1["extra"] += 10**-4 * numpy.random.random(size=size)
table1.to_csv(headerfile_extra_cols_diff,index=False,header=True,sep="\t")
table1.to_csv(noheaderfile_extra_cols_diff,index=False,header=False,sep="\t",
columns=["extra"]+keyorder)
headerfile_extra_cols_diff.close()
noheaderfile_extra_cols_diff.close()
shufidx = numpy.arange(size)
shuffle(shufidx)
table2 = pd.DataFrame({ K : V[shufidx] for K,V in cols.items()})
table2.to_csv(headerfile_shuffled,index=False,header=True,sep="\t")
table2.to_csv(noheaderfile_shuffled,index=False,header=False,sep="\t",
columns=keyorder)
headerfile_shuffled.close()
noheaderfile_shuffled.close()
table2["extra"] = table1["extra"][shufidx]
table2.to_csv(headerfile_extra_cols_shuffled,index=False,header=True,sep="\t")
table2.to_csv(noheaderfile_extra_cols_shuffled,
index=False,header=False,sep="\t",
columns=["extra"]+keyorder)
headerfile_extra_cols_shuffled.close()
noheaderfile_extra_cols_shuffled.close()
# Define tests, as tuples of:
# -Test name/description
# -Command-line arguments to pass to :py:mod:`plastid.bin.test_table_equality`
# -Expected exit code/returns status for :py:func:`main`
tests = [
("same",
"%s %s" % (headerfile.name,headerfile.name),
0),
("diff_column_names",
"%s %s" % (headerfile.name,headerfile_extra_cols.name),
1),
("extra_column_names_ignored",
"%s %s --exclude extra" % (headerfile.name,headerfile_extra_cols.name),
0),
("shuffled_rows",
"%s %s" % (headerfile.name,headerfile_shuffled.name),
1),
("shuffled_rows_name_sort",
"%s %s --sort_keys idxA" % (headerfile.name,headerfile_shuffled.name),
0),
("shuffled_rows_multi_name_sort",
"%s %s --sort_keys strB chrA" % (headerfile.name,headerfile_shuffled.name),
0),
("same_column_names_diff_values",
"%s %s" % (headerfile_extra_cols.name,headerfile_extra_cols_diff.name),
1),
("same_column_names_diff_values_tol",
"%s %s --tol 0.01" % (headerfile_extra_cols.name,headerfile_extra_cols_diff.name),
0),
("same_column_names_diff_values_ignored",
"%s %s --exclude extra" % (headerfile_extra_cols.name,headerfile_extra_cols_diff.name),
0),
("shuffled_rows_extra_columns_ignored",
"%s %s --exclude extra" % (headerfile.name,headerfile_extra_cols_shuffled.name),
1),
("shuffled_rows_extra_columns_ignored_name_sort",
"%s %s --exclude extra --sort_keys idxA" % (headerfile.name,headerfile_extra_cols_shuffled.name),
0),
("noheader_same",
"%s %s --no_header" % (noheaderfile.name,noheaderfile.name),
0),
("noheader_extra_columns",
"%s %s --no_header" % (noheaderfile.name,noheaderfile_extra_cols.name),
1),
("noheader_shuffled_rows",
"%s %s --no_header" % (noheaderfile.name,noheaderfile_shuffled.name),
1),
("noheader_shuffled_rows_int_sort",
"%s %s --no_header --sort_keys 0" % (noheaderfile.name,noheaderfile_shuffled.name),
0),
("noheader_diff_values",
"%s %s --no_header" % (noheaderfile_extra_cols.name,noheaderfile_extra_cols_diff.name),
1),
("noheader_diff_values_tol",
"%s %s --no_header --tol 0.01" % (noheaderfile_extra_cols.name,noheaderfile_extra_cols_diff.name),
0),
("no_header_diff_values_ignored",
"%s %s --no_header --exclude 0" % (noheaderfile_extra_cols.name,noheaderfile_extra_cols_diff.name),
0),
("no_header_shuffled_rows_extra_columns_ignored_int_sort",
"%s %s --no_header --exclude 0 --sort_keys 1" % (noheaderfile_extra_cols.name,noheaderfile_extra_cols_shuffled.name),
0)
]
""" Tests to conduct, as tuples of:
- Test name/description
- Command-line arguments to pass to :py:mod:`plastid.bin.test_table_equality`
- Expected exit code/returns status for :py:func:`main`
"""
for test_name, argstr, expected_exit in tests:
yield check_exit_status, test_name, argstr, expected_exit
# clean up
os.unlink(headerfile.name )
os.unlink(headerfile_extra_cols.name )
os.unlink(headerfile_extra_cols_diff.name )
os.unlink(headerfile_extra_cols_shuffled.name )
os.unlink(headerfile_shuffled.name )
os.unlink(headerfile_diff_vals.name )
os.unlink(noheaderfile.name )
os.unlink(noheaderfile_extra_cols.name )
os.unlink(noheaderfile_extra_cols_diff.name )
os.unlink(noheaderfile_extra_cols_shuffled.name )
os.unlink(noheaderfile_shuffled.name )
os.unlink(noheaderfile_diff_vals.name )
cleanup_resources()
def main():
import warnings
warnings.filterwarnings('ignore')
ap = argparse.ArgumentParser()
ap.add_argument("infile", type=str,
help="file to process")
ap.add_argument("-f", dest="rules",
default= resource_filename(__name__, RULESFILE),
help="name of file containing transformation rules" +
" (default: {})".format(RULESFILE))
ap.add_argument("-o", dest="outfile", default=OUTFILE,
help="destination file" +
" (default: {})".format(OUTFILE))
ap.add_argument("-m", dest="marker", default="",
help="marker to be added to new nodes and edges" +
" (default: None)")
args = ap.parse_args()
with open(args.outfile, "w") as outfile:
# write declaration
# write open corpus tag
print('<?xml version="1.0"',
'encoding="ISO-8859-1"',
'standalone="yes"?>',
file=outfile)
print('<corpus',
'id="Release2_1-{:%y%m%d}">'.format(date.today()),
file=outfile)
_, head = next(etree.iterparse(args.infile,
tag="head",
remove_blank_text=True))
# if marker requested
# insert test nodes into features section in header
if args.marker:
new_nodes_list = new_node_types(args.rules, args.marker)
cat_features = head.find("annotation/feature[@name='cat']")
add_new_types(cat_features, new_nodes_list)
new_edges_list = new_edge_types(args.rules, args.marker)
edge_features = head.find("annotation/edgelabel")
add_new_types(edge_features, new_edges_list)
secedge_features = head.find("annotation/secedgelabel")
add_new_types(secedge_features, new_edges_list)
else:
new_nodes_list = None
new_edges_list = None
# write head data
print(str(etree.tostring(head, pretty_print=True), "utf-8"),
file=outfile)
# get all rules
all_rules = list(load_rules(args.rules, args.infile,
new_nodes=new_nodes_list,
new_edges=new_edges_list,
_marker=args.marker))
# write open body tag
print("<body>", file=outfile)
# iterate through all sentences
# based on fast_iter from:
# http://www.ibm.com/developerworks/xml/library/x-hiperfparse/
for _, element in etree.iterparse(args.infile,
tag=SENTENCE_TAG,
remove_blank_text=True):
sent = process_sentence(element, all_rules)
print(str(etree.tostring(sent, pretty_print=True), "iso-8859-1"),
file=outfile)
element.clear()
while element.getprevious() is not None:
del element.getparent()[0]
# write close body tag and close corpus tag
print("</body>\n</corpus>", file=outfile)
print("\ndone!")
cleanup_resources()
exit()
"""
Utility functions
"""
import rastercube
import numpy as np
import os
import errno
from datetime import datetime
import calendar
import cPickle as pickle
import pkg_resources
import atexit
# Cleanup tmpdir used by asset_fname on interpreter exit
atexit.register(lambda: pkg_resources.cleanup_resources())
def asset_fname(relpath):
"""
Gets the filename to an asset relative to the rastercube package root.
When rastercube is packaged as an egg, you can't access assets using
os.path.join(rastercube.__file__, 'assets/foo.json') since the egg is a
zip. So you should use this function.
See :
http://peak.telecommunity.com/DevCenter/PythonEggs#accessing-package-resources
>>> fname = asset_fname('assets/modis_tiles.geojson')
"""
return pkg_resources.resource_filename(rastercube.__name__, relpath)
