def _get_pki_test_data_path(): try: return resource_filename( Requirement.parse("n6lib"), 'n6lib/tests/certs_and_requests_for_testing/') finally: cleanup_resources()
def tearDown(self): """ Clean Up """ rmtree(self._tmp_base_folder) # pkg_resources.resources_filename cashes files, when # in a compiled distribution. We need to clean up potentially # created files. pkg_resources.cleanup_resources()
def start(): """Start client.""" sslcontext = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) sslcontext.verify_mode = ssl.CERT_REQUIRED req = Requirement.parse("all1input") sslcontext.load_cert_chain( certfile=resource_filename( req, "all1input/{}.crt".format(c.cert_name)), keyfile=resource_filename( req, "all1input/{}.key".format(c.cert_name))) sslcontext.load_verify_locations(resource_filename( req, "all1input/{}.pem".format(c.root_cert_name))) loop = asyncio.get_event_loop() try: while True: try: coro = loop.create_connection( partial(All1InputClientProtocol, loop), c.ip, c.port, ssl=sslcontext) loop.run_until_complete(coro) loop.run_forever() except ConnectionRefusedError: pass sleep(5) except KeyboardInterrupt: pass finally: loop.close() cleanup_resources()
def tearDown(self): """ Clean up the sandbox. """ rmtree(self._test_sandbox, ignore_errors=True) # pkg_resources.resources_filename cashes files, when # in a compiled distribution. We need to clean up potentially # created files. pkg_resources.cleanup_resources()
def _terminate(self, restart=False, app=None): log.info("Terminating Stoq") # This removes all temporary files created when calling # get_resource_filename() that extract files to the file system import pkg_resources pkg_resources.cleanup_resources() log.debug('Stopping deamon') from stoqlib.lib.daemonutils import stop_daemon stop_daemon() # Finally, go out of the reactor and show possible crash reports yield self._quit_reactor_and_maybe_show_crashreports() if restart: from stoqlib.lib.process import Process log.info('Restarting Stoq') args = [sys.argv[0], '--no-splash-screen'] if app is not None: args.append(app) Process(args) # os._exit() forces a quit without running atexit handlers # and does not block on any running threads # FIXME: This is the wrong solution, we should figure out why there # are any running threads/processes at this point log.debug("Terminating by calling os._exit()") os._exit(0) raise AssertionError("Should never happen")
def run(param_file, flag_ow=None): """ Main function for HYVR generation Parameters: param_file (str): Parameter file location flag_ow : bool If True, an existing run directory is overwritten. Returns: Save data outputs as parameter file """ # TODO: implement a seed option to get the same model # np.random.seed(84) # Load parameter file run, model, hydraulics = hp.model_setup(param_file) hp.set_up_directories(run, param_file, flag_ow) for sim in range(1, int(run['numsim'])+1): # Generate facies model.generate_model() # model.assign_facies_azim_dip() assign_facies_azim_dip(model) # return if model.generate_hydraulics is True: # Generate internal heterogeneity model.generate_hydraulic_parameters(hydraulics) # from hyvr.postprocess.plotting import cross_section_pcolor # cross_section_pcolor(model, 'k_iso', log=True, y=10, xlim=[15.5, 16.4], ylim=[7.1, 7.7] # cross_section_pcolor(model, 'facies', log=False, y=10, xlim=[15.5, 16.4], ylim=[7.1, 7.7]) # cross_section_pcolor(model, 'ha', log=False, y=10, xlim=[15.5, 16.4], ylim=[7.1, 7.7]) # cross_section_pcolor(model, 'hat', log=False, y=10, xlim=[15.5, 16.4], ylim=[7.1, 7.7]) # cross_section_pcolor(model, 'ae', log=False, y=10, xlim=[15.5, 16.4], ylim=[7.1, 7.7]) # cross_section_pcolor(model, 'strata', log=False, y=10, xlim=[15.5, 16.4], ylim=[7.1, 7.7]) # cross_section_pcolor(model, 'dip') # cross_section_pcolor(model, 'azim') # Save data if run['numsim'] > 1: realname = 'real_{:03d}'.format(sim) realdir = os.path.join(run['rundir'], realname) else: realname = run['runname'] realdir = run['rundir'] hu.try_makefolder(realdir) create_outputs(model, realdir, realname, run['outputs']) if param_file == 0: # this is just the testcase, so we remove the output from pkg_resources import cleanup_resources cleanup_resources() import shutil runpath = os.path.abspath(run['rundir']) if os.path.exists(runpath): shutil.rmtree(runpath)
def start(): """Start Web server.""" config = OrganiserConfiguration() asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) app = web.Application(debug=config.server.debug) aiohttp_jinja2.setup( app, loader=jinja2.FileSystemLoader( resource_filename(Requirement.parse('organiser'), 'organiser/templates')), auto_reload=config.server.debug, context_processors=[aiohttp_login.flash.context_processor]) aiohttp_session.setup( app, EncryptedCookieStorage(config.server.session_secret, max_age=config.server.session_max_age)) app.middlewares.append(aiohttp_login.flash.middleware) app['db_client'] = AsyncIOMotorClient() app['db'] = app['db_client'].organiser auth_settings = {} for setting in config.authentication: auth_settings[setting.upper()] = config.authentication[setting] aiohttp_login.setup(app, MotorStorage(app['db']), auth_settings) app.router.add_static("/static/", path=Path(__file__) / ".." / "static", name="static") app.router.add_get("/", index_handler) app.router.add_get("/calendar/{period}/", calendar_handler) app.router.add_get("/calendar/{period}/{date}/", calendar_handler) app.router.add_get("/lists/", list_handler) web.run_app(app, host=config.server.ip, port=config.server.port) cleanup_resources()
def tearDown(self): extractor.registered_extractors = [] pkg_resources.cleanup_resources() shutil.rmtree(self.target_path) if self.site.source: self.site.source.stoppedEvent(self.site.source, None) Logger.stop()
def _terminate(self, restart=False, app=None): log.info("Terminating Stoq") # This removes all temporary files created when calling # get_resource_filename() that extract files to the file system import pkg_resources pkg_resources.cleanup_resources() log.debug('Stopping deamon') from stoqlib.lib.daemonutils import stop_daemon stop_daemon() # Finally, go out of the reactor and show possible crash reports log.debug("Show some crash reports") self._show_crash_reports() log.debug("Stoq gtk.main") import gtk gtk.main_quit() if restart: from stoqlib.lib.process import Process log.info('Restarting Stoq') args = [sys.argv[0], '--no-splash-screen'] if app is not None: args.append(app) Process(args) # os._exit() forces a quit without running atexit handlers # and does not block on any running threads # FIXME: This is the wrong solution, we should figure out why there # are any running threads/processes at this point log.debug("Terminating by calling os._exit()") os._exit(0) raise AssertionError("Should never happen")
def read_config() -> dict: """ reads the config json file :return: the contents of the config file """ try: path_components = __name__.split('.')[:-1] resource_path = '.'.join(path_components) cached_file = pkg_resources.resource_filename( resource_path, 'app.prop') # check if the file exists at the path. file_contents = json.load(open(cached_file)) # clean up all resources pkg_resources.cleanup_resources() # return the file contents return file_contents except FileNotFoundError: print('Could not locate the "app.prop". ' 'Please make sure it exists in the same directory.') except json.JSONDecodeError: print("The configuration file must have a valid JSON.") except TypeError: print("The configuration file must have a valid JSON.")
def tearDownClass(cls): """ Removes test temp directory and deletes all files :return: """ pkg_resources.cleanup_resources() shutil.rmtree(cls.test_dir) # remove temp directory and delete all files
def tearDown(self): """Clean up after each test.""" pkg_resources.cleanup_resources() assert(self.tmpfhandle is None) if self.tmpfname is not None: os.unlink(self.tmpfname) self.tmpfname = None
def tearDownClass(cls): pkg_resources.cleanup_resources() for d in [ cls.prediction_directory, cls.prediction_directory_2, cls.prediction_directory_3, cls.groundtruth_directory, cls.groundtruth_2_directory ]: shutil.rmtree(d)
def get_config_base_lines(): try: filename = resource_filename(Requirement.parse('n6sdk'), 'n6sdk/_api_test_tool/config_base.ini') with open(filename) as f: return f.read().splitlines() finally: cleanup_resources()
def iter_config_base_lines(): try: filename = resource_filename(Requirement.parse('n6sdk-py2'), 'n6sdk/_api_test_tool/config_base.ini') with open(filename, 'rb') as f: for line in f.read().splitlines(): yield line.decode('utf-8') finally: cleanup_resources()
def main(): """main.""" demo_path = 'funkload-demo' print "Extract FunkLoad examples into ./%s : ... " % demo_path, cache_path = resource_filename('funkload', 'demo') demo_path = os.path.join(os.path.abspath(os.path.curdir), demo_path) copytree(cache_path, demo_path) cleanup_resources() print "done."
def execute_freesasa_api(structure): """ Calls freesasa using its Python API and returns per-residue accessibilities. """ try: from freesasa import Classifier, structureFromBioPDB, calc except ImportError as err: print( '[!] The binding affinity prediction tools require the \'freesasa\' Python API', file=sys.stderr) raise ImportError(err) asa_data, rsa_data = {}, {} _rsa = rel_asa['total'] config_path = os.environ.get( 'FREESASA_PAR', pkg_resources.resource_filename('prodigy', 'naccess.config')) classifier = Classifier(config_path) pkg_resources.cleanup_resources() # classifier = freesasa.Classifier( os.environ["FREESASA_PAR"]) # Disable with stdchannel_redirected(sys.stderr, os.devnull): try: struct = structureFromBioPDB( structure, classifier, ) result = calc(struct) except AssertionError as e: error_message = '\n[!] Error when running freesasa: \n[!] {}'.format( e) print(error_message) raise Exception(error_message) # iterate over all atoms to get SASA and residue name for idx in range(struct.nAtoms()): atname = struct.atomName(idx) resname = struct.residueName(idx) resid = struct.residueNumber(idx) chain = struct.chainLabel(idx) at_uid = (chain, resname, resid, atname) res_uid = (chain, resname, resid) asa = result.atomArea(idx) asa_data[at_uid] = asa # add asa to residue rsa_data[res_uid] = rsa_data.get(res_uid, 0) + asa # convert total asa ro relative asa rsa_data.update( (res_uid, asa / _rsa[res_uid[1]]) for res_uid, asa in rsa_data.items()) return asa_data, rsa_data
def start(): """Start server.""" global loop, ignore_devices, all_devices, STOP, devices req = Requirement.parse("all1input") server_ssl = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH) server_ssl.verify_mode = ssl.CERT_REQUIRED server_ssl.load_cert_chain( certfile=resource_filename( req, "all1input/{}.crt".format(c.server_cert_name)), keyfile=resource_filename( req, "all1input/{}.key".format(c.server_cert_name))) server_ssl.load_verify_locations(resource_filename( req, "all1input/{}.pem".format(c.root_cert_name))) client_ssl = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) client_ssl.verify_mode = ssl.CERT_REQUIRED client_ssl.load_cert_chain( certfile=resource_filename( req, "all1input/{}.crt".format(c.cert_name)), keyfile=resource_filename( req, "all1input/{}.key".format(c.cert_name))) client_ssl.load_verify_locations(resource_filename( req, "all1input/{}.pem".format(c.root_cert_name))) all_devices = [evdev.InputDevice(fn) for fn in evdev.list_devices()] devices = {} ignore_devices = [] loop = asyncio.get_event_loop() coro_server = loop.create_server( All1InputServerClientProtocol, c.ip, c.port, ssl=server_ssl) server = loop.run_until_complete(coro_server) coro_client = loop.create_connection( partial(All1InputClientProtocol, loop), c.ip, c.port, ssl=client_ssl) loop.run_until_complete(coro_client) loop.call_later(1, update_devices) loop.call_later(0.01, move_mouse) try: loop.run_forever() except KeyboardInterrupt: STOP = True finally: for dev in devices: devices[dev].ungrab() devices[dev].close() server.close() loop.run_until_complete(server.wait_closed()) loop.close() cleanup_resources()
def load_prefs(): prefs_file = os.path.join(os.path.expanduser('~'), '.dapplerc') if not os.path.exists(prefs_file): shutil.copy(resource_filename(__name__, 'defaults/_dapplerc'), prefs_file) cleanup_resources() with open(prefs_file, 'r') as f: return yaml.load(f)
async def _exec_on_midi_end(proc: Process, func: Callable, port: str) -> int: """Exec func when midi stops playing.""" return_code = await proc.wait() if proc.stdout is None: reset_proc = await create_subprocess_exec( 'pmidi', '-p', port, '-d', '0', resource_filename(Requirement.parse("voicetrainer"), 'voicetrainer/reset.midi')) await reset_proc.wait() cleanup_resources() _PROC_LOCK.release() await func() return return_code
def execute(self): """ Run the scanner """ try: self._start_zap() if not self._wait_for_zap_start(): log.error("ZAP failed to start") error = Error( tool=self.get_name(), error="ZAP failed to start", details="ZAP daemon failed to start" ) self.errors.append(error) return log.info("Target: %s", self.config.get("target")) self._prepare_context() self._setup_scan_policy() self._spider() self._wait_for_passive_scan() self._ajax_spider() self._wait_for_passive_scan() self._active_scan() self._wait_for_passive_scan() except: log.exception("Exception during ZAP scanning") error = Error( tool=self.get_name(), error=f"Exception during ZAP scanning", details=f"```\n{traceback.format_exc()}\n```" ) self.errors.append(error) finally: try: # Get report log.info("Getting ZAP report") zap_report = self._zap_api.core.jsonreport() # Parse JSON log.info("Processing findings") parse_findings(zap_report, self) except: log.exception("Exception during ZAP findings processing") error = Error( tool=self.get_name(), error=f"Exception during ZAP findings processing", details=f"```\n{traceback.format_exc()}\n```" ) self.errors.append(error) self._save_intermediates() pkg_resources.cleanup_resources() self._stop_zap()
def init(): shutil.copytree(resource_filename(__name__, 'defaults/_dapple'), 'dapple') # resource_filename leaves behind temp files. # cleanup_resources deletes them. cleanup_resources() try: load_plugins() (dapple.plugins.load('core.install_package'))( 'dapple', ipfs='QmU9tDS5UebJ3mhakB2GtiJ3E2zbn2b1xaXzUNkig9Qw41') except: print("ERROR: Could not pull `dapple` package from IPFS! " "You might try installing it manually.", file=sys.stderr) print("Init'ed Dapple package. You might want to edit dapple/dappfile now.")
def make_profile(): """ Create a local profile in user's home folder. """ home_path = os.path.expanduser('~/.ScatPy/') try: os.makedirs(home_path) except OSError: raise(IOError('A folder ~./ScatPy already exists. Delete it manually before proceeding.')) files = resource_listdir('ScatPy', 'profiles') for f in files: src = resource_filename('ScatPy', os.path.join('profiles', f)) shutil.copy(src, home_path) cleanup_resources()
def init(): shutil.copytree(resource_filename(__name__, 'defaults/_dapple'), '.dapple') # resource_filename leaves behind temp files. # cleanup_resources deletes them. cleanup_resources() try: load_plugins() (dapple.plugins.load('core.install_package'))( 'core', ipfs='QmP3sVwo7nRxekM37pyLL4AS7E1rX5qnAnFNGLpgwgZsHj') except: print("ERROR: Could not pull `core` package from IPFS! " "You might try installing it manually.", file=sys.stderr) print("Init'ed Dapple package. You might want to edit .dapple/dappfile now.")
def make_profile(): """ Create a local profile in user's home folder. """ home_path = os.path.expanduser('~/.ScatPy/') try: os.makedirs(home_path) except OSError: raise IOError files = resource_listdir('ScatPy', 'profiles') for f in files: src = resource_filename('ScatPy', os.path.join('profiles', f)) shutil.copy(src, home_path) cleanup_resources()
def predict(self, document): """ tag the new data basde on a given model :param document: document object :returns: list of predictions """ logger.info('Start to predict') y,x = self.fit(document) if Svm._auto_config: file_name = self._auto_load('file', self.model_file) m = svmutil.svm_load_model(file_name) pkg_resources.cleanup_resources() else : m = svmutil.svm_load_model(self.model_file) y_pred, p_acc, p_val = svmutil.svm_predict(y, x, m, "-q") return y_pred
def init(): shutil.copytree(resource_filename(__name__, 'defaults/_dapple'), 'dapple') # resource_filename leaves behind temp files. # cleanup_resources deletes them. cleanup_resources() try: load_plugins() (dapple.plugins.load('core.install_package'))( 'dapple', ipfs='QmU9tDS5UebJ3mhakB2GtiJ3E2zbn2b1xaXzUNkig9Qw41') except: print( "ERROR: Could not pull `dapple` package from IPFS! " "You might try installing it manually.", file=sys.stderr) print( "Init'ed Dapple package. You might want to edit dapple/dappfile now.")
def make_profile(): """ Create a local profile in user's home folder. """ home_path = os.path.expanduser('~/.ScatPy/') try: os.makedirs(home_path) except OSError: raise (IOError( 'A folder ~./ScatPy already exists. Delete it manually before proceeding.' )) files = resource_listdir('ScatPy', 'profiles') for f in files: src = resource_filename('ScatPy', os.path.join('profiles', f)) shutil.copy(src, home_path) cleanup_resources()
def _terminate(self, restart=False, app=None): log.info("Terminating Stoq") # This removes all temporary files created when calling # get_resource_filename() that extract files to the file system import pkg_resources pkg_resources.cleanup_resources() log.debug('Stopping deamon') from stoqlib.lib.daemonutils import stop_daemon stop_daemon() # Finally, go out of the reactor and show possible crash reports log.debug("Show some crash reports") self._show_crash_reports() # Make sure that no connection is left open (specially on Windows) try: from stoqlib.database.runtime import get_default_store get_default_store().close() except Exception: pass if restart: from stoqlib.lib.process import Process log.info('Restarting Stoq') args = [sys.argv[0], '--no-splash-screen'] if app is not None: args.append(app) Process(args) # os._exit() forces a quit without running atexit handlers # and does not block on any running threads # FIXME: This is the wrong solution, we should figure out why there # are any running threads/processes at this point log.debug("Terminating by calling os._exit()") os._exit(0) raise AssertionError("Should never happen")
def start(): """Start Web server.""" config = DndConfiguration() # asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) app = web.Application(debug=config.server.debug) aiohttp_jinja2.setup( app, loader=jinja2.FileSystemLoader( resource_filename(Requirement.parse('dnd'), 'dnd/templates')), auto_reload=config.server.debug, context_processors=[aiohttp_login.flash.context_processor]) aiohttp_session.setup( app, EncryptedCookieStorage(config.server.session_secret, max_age=config.server.session_max_age)) aiohttp_jinja2.get_env(app).filters['to_roman'] = toRoman aiohttp_jinja2.get_env(app).filters['max'] = max aiohttp_jinja2.get_env(app).filters['cutoff_dict'] = _cutoff_dict_filter app.middlewares.append(aiohttp_login.flash.middleware) app['db_client'] = AsyncIOMotorClient() app['db'] = app['db_client'].dnd auth_settings = {} for setting in config.authentication: auth_settings[setting.upper()] = config.authentication[setting] aiohttp_login.setup(app, MotorStorage(app['db']), auth_settings) app.router.add_static("/static/", path=Path(__file__) / ".." / "static", name="static") app.router.add_get("/", index_handler) app.router.add_post("/api/new-character/", new_character_data_handler) app.router.add_get("/{id}/{name}/", character_handler) app.router.add_post("/api/{id}/{attribute}/{extra}/", data_handler) app.router.add_post("/api/{id}/{attribute}/", data_handler) web.run_app(app, host=config.server.ip, port=config.server.port) cleanup_resources()
def execute(self): """ Run the scanner """ # Get config config_data = pkg_resources.resource_string( "dusty", f"{'/'.join(__name__.split('.')[1:-1])}/data/w3af_full_audit.w3af" ) if self.config.get("config_file", None): with open(self.config.get("config_file"), "r") as config: config_data = config.read() # Make temporary files config_file_fd, config_file = tempfile.mkstemp() output_file_fd, output_file = tempfile.mkstemp() log.debug("Config file: %s", config_file) log.debug("Output file: %s", output_file) # Fill config data variables config_data = config_data.decode("utf-8").format( target=self.config.get("target"), output_section=constants.W3AF_OUTPUT_SECTION.replace("{output_file}", output_file) ) os.write(config_file_fd, config_data.encode("utf-8")) # Close unneeded handles os.close(config_file_fd) os.close(output_file_fd) # Run scanner task = subprocess.run([ "w3af_console", "-y", "-n", "-s", config_file ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) log.log_subprocess_result(task) # Parse findings parse_findings(output_file, self) # Save intermediates self.save_intermediates(output_file, config_file, task) # Remove temporary files os.remove(config_file) os.remove(output_file) pkg_resources.cleanup_resources()
def main(): args = parse_arguments() try: with tempfile.TemporaryDirectory() as temp_dir: backend = os.path.join(get_resource_dir('drawio/war'), 'export2.html') frontend = os.path.join(temp_dir, 'frontend.js') with open(frontend, 'wb') as frontend_file: in_data = get_resource('frontend.js.in') in_data = in_data.replace( b'@FILE@', bytearray("'" + backend + "'", 'ASCII')) frontend_file.write(in_data) # chop newlines from input processed_input = os.path.join(temp_dir, 'input') with open(processed_input, 'w') as in_file: data = args.input.readlines() data = [l.strip() for l in data] in_file.write(''.join(data)) with open(processed_input, 'r') as in_file: command = [args.phantomjs.name, frontend, args.format, str(args.scale), str(args.quality)] subprocess.check_call( command, stdout=args.output, stdin=in_file) args.output.close() finally: pkg_resources.cleanup_resources()
def start(): """Start cert manager.""" parser = argparse.ArgumentParser( prog="cert_manager", description='Create certificates for all1input.') parser.add_argument( 'action', choices=["all", "client"], help=("all: create root, server and client cert from config |" "client: creates client certificate with NAME")) parser.add_argument("--name", default="client", help="certificate name to use for client certificate") args = parser.parse_args() if args.action == "all": try: print("Making root CA") create_root_ca(c.root_cert_name) except FileExists: print("Root CA already exists") finally: cleanup_resources() try: print("Making server certificate") create_certificate(c.server_cert_name, True, c.root_cert_name, c.ip) except FileExists: print("Server cert already exists") finally: cleanup_resources() try: print("Making client certificate") create_certificate(c.cert_name, False, c.root_cert_name) except FileExists: print("Client cert already exists") finally: cleanup_resources() elif args.action == "client": try: print("Making client certificate") create_certificate(args.name, False, c.root_cert_name) finally: cleanup_resources()
def tearDownModule(): """Remove test dataset files after unit tests are complete""" if test_info["temp_file_path"] != "": shutil.rmtree(test_info["temp_file_path"]) cleanup_resources()
def tearDownClass(cls): """Clean up temp files after tests are complete""" cleanup_resources() shutil.rmtree(cls.optdict["outdir"])
def stats(self, resultset=()): slices = OrderedDict() datadict = defaultdict(list) # 1: Create a giant RDF graph consisting of all triples of all # repos' commondata. To avoid parsing the same RDF files # over and over, this section duplicates the logic of # DocumentRepository.commondata to make sure each RDF # file is loaded only once. ttlfiles = set() resource_graph = Graph() namespaces = {} for repo in self.repos: for prefix, ns in repo.make_graph().namespaces(): assert ns not in namespaces or namespaces[ ns] == prefix, "Conflicting prefixes for ns %s" % ns namespaces[ns] = prefix resource_graph.bind(prefix, ns) for cls in inspect.getmro(repo.__class__): if hasattr(cls, "alias"): commonpath = "res/extra/%s.ttl" % cls.alias if os.path.exists(commonpath): ttlfiles.add(commonpath) elif pkg_resources.resource_exists( 'ferenda', commonpath): ttlfiles.add( pkg_resources.resource_filename( 'ferenda', commonpath)) self.log.debug( "stats: Loading resources %s into a common resource graph" % list(ttlfiles)) for filename in ttlfiles: resource_graph.parse(data=util.readfile(filename), format="turtle") pkg_resources.cleanup_resources() # 2: if used in the resultset mode, only calculate stats for those # resources/documents that are in the resultset. resultsetmembers = set() if resultset: for r in resultset: resultsetmembers.add(r['iri']) # 3: using each repo's faceted_data and its defined facet # selectors, create a set of observations for that repo # # FIXME: If in resultset mode, we might ask a repo for its # faceted data and then use exactly none of it since it # doesn't match anything in resultsetmembers. We COULD analyze # common resultset iri prefixes and then only call # faceted_data for some (or one) repo. for repo in self.repos: data = repo.faceted_data() if resultsetmembers: data = [r for r in data if r['uri'] in resultsetmembers] for facet in repo.facets(): if not facet.dimension_type: continue dimension, obs = self.stats_slice(data, facet, resource_graph) if dimension in slices: # since observations is a Counter not a regular # dict, if slices[dimensions] and observations # have common keys this will add the counts not # replace them. slices[dimension].update(obs) else: slices[dimension] = obs # 4. Transform our easily-updated data structures to the list # of dicts of lists that we're supposed to return. res = {"type": "DataSet", "slices": []} for k, v in sorted(slices.items()): observations = [] for ok, ov in sorted(v.items()): observations.append({ok[0]: ok[1], "count": ov}) res['slices'].append({ "dimension": k, "observations": observations }) return res
log_exception("Shutdown failed.", Logtype.ERROR) finally: seconds_left = 30 while seconds_left > 0 and threading.active_count() > 2: #main + logging thread time.sleep(1) seconds_left -= 1 if threading.active_count() > 2: #main + logging thread print("Hanging threads:") for t in threading.enumerate(): if not(t == threading.current_thread()) and t.isAlive(): print("* " + t.name) print("Force quit.") Logger.stop() sys.exit(0) #TODO: does this really kill all threads? Logger.stop() pkg_resources.cleanup_resources(force=False) def signal_handler(signal, frame): log("Shutdown requested by user.", Logtype.INFO) shutdown() def log(msg, logtype = Logtype.NONE): Logger.log("witica", msg, logtype) def log_exception(msg, logtype = Logtype.NONE, exc_info=None): Logger.log_exception("witica", msg, logtype, exc_info) def create_source(args): if args.source: return Source.construct_from_file(args.source) else:
def tearDownModule(): """Remove test dataset files after unit tests are complete""" cleanup_resources()
def tearDown(self): extractor.registered_extractors = [] pkg_resources.cleanup_resources() Logger.stop()
def tearDownModule(): cleanup_resources()
def close(self): self.db.close() super().close() pkg_resources.cleanup_resources()
def stats(self, resultset=()): slices = OrderedDict() datadict = defaultdict(list) # 1: Create a giant RDF graph consisting of all triples of all # repos' commondata. To avoid parsing the same RDF files # over and over, this section duplicates the logic of # DocumentRepository.commondata to make sure each RDF # file is loaded only once. ttlfiles = set() resource_graph = Graph() namespaces = {} for repo in self.repos: for prefix, ns in repo.make_graph().namespaces(): assert ns not in namespaces or namespaces[ns] == prefix, "Conflicting prefixes for ns %s" % ns namespaces[ns] = prefix resource_graph.bind(prefix, ns) for cls in inspect.getmro(repo.__class__): if hasattr(cls, "alias"): commonpath = "res/extra/%s.ttl" % cls.alias if os.path.exists(commonpath): ttlfiles.add(commonpath) elif pkg_resources.resource_exists('ferenda', commonpath): ttlfiles.add(pkg_resources.resource_filename('ferenda', commonpath)) self.log.debug("stats: Loading resources %s into a common resource graph" % list(ttlfiles)) for filename in ttlfiles: resource_graph.parse(data=util.readfile(filename), format="turtle") pkg_resources.cleanup_resources() # 2: if used in the resultset mode, only calculate stats for those # resources/documents that are in the resultset. resultsetmembers = set() if resultset: for r in resultset: resultsetmembers.add(r['iri']) # 3: using each repo's faceted_data and its defined facet # selectors, create a set of observations for that repo # # FIXME: If in resultset mode, we might ask a repo for its # faceted data and then use exactly none of it since it # doesn't match anything in resultsetmembers. We COULD analyze # common resultset iri prefixes and then only call # faceted_data for some (or one) repo. for repo in self.repos: data = repo.faceted_data() if resultsetmembers: data = [r for r in data if r['uri'] in resultsetmembers] for facet in repo.facets(): if not facet.dimension_type: continue dimension, obs = self.stats_slice(data, facet, resource_graph) if dimension in slices: # since observations is a Counter not a regular # dict, if slices[dimensions] and observations # have common keys this will add the counts not # replace them. slices[dimension].update(obs) else: slices[dimension] = obs # 4. Transform our easily-updated data structures to the list # of dicts of lists that we're supposed to return. res = {"type": "DataSet", "slices": [] } for k, v in sorted(slices.items()): observations = [] for ok, ov in sorted(v.items()): observations.append({ok[0]: ok[1], "count": ov}) res['slices'].append({"dimension": k, "observations": observations}) return res
def main(): ap = argparse.ArgumentParser() ap.add_argument('source', type=str, choices=['twitter', 'folder', 'txtfile', 'stdin', 'demo'], help='Where the input data should come from.') ap.add_argument('query', type=str, nargs=argparse.REMAINDER, help='Query for Twitter search.') ap.add_argument('-p', '--percent', type=float, default=0.25, help="Percentage of clusters to return as main topics.") ap.add_argument('-n', '--number', type=int, default=2, help="Number of representative sentences from each topic.") ap.add_argument('-t', '--no-tokenize', action='store_true', help="When present, don't tokenize input string.") args = ap.parse_args() print("reading source text...", "({})".format(args.source)) if args.source == 'twitter': print("with query:", args.query) input_text = search_twitter(args.query) elif args.source == 'folder': input_text = "" path = args.query[0] for resfile in next(os.walk(path))[2]: input_text += open(path + '/' + resfile, encoding='utf-8').read() elif args.source == 'txtfile': input_text = open(args.query[0], "r", encoding="utf-8").read() elif args.source == 'stdin': input_text = sys.stdin.read() elif args.source == 'demo': input_text = demo() else: print(args.usage) sys.exit(1) print("tokenizing...") if not args.no_tokenize: tokenized_input = pre.tokenizes(input_text) print("tagging...") if not args.no_tokenize: tagged_input = pre.pos_tag(tokenized_input) else: tagged_input = pre.pos_tag(input_text.encode()) print("generating sentence objects... ", end="") sentences = pre.make_sent_objs(tagged_input) print("({} sentences)".format(len(sentences))) print("computing matrix for clustering...") X = clst.build_sent_matrix(sentences, weighting='binary') print("clustering...") clusters, eps = clst.get_clusters(X, sentences, min_samp=int(len(sentences)*0.04)) print("\u03B5 set to", eps) print(clst.cluster_sizes(clusters)) print("selecting top sentences...") num_to_take = int(args.percent * len(clusters)) if num_to_take < 2: num_to_take = len(clusters) toptop = sel.top_n_topics(num_to_take, clusters, len(sentences)) # output result print("result:") print_toptopics(toptop, sentences, clusters, args.number) cleanup_resources() exit(0)
def tearDown(self): pkg_resources.cleanup_resources()
def tearDownClass(cls): pkg_resources.cleanup_resources() shutil.rmtree(cls.prediction_directory)
def zap(config): # Nested functions def _wait_for_completion(condition, status, message, interval=10): """ Watch progress """ current_status = status() logging.info(message, current_status) while condition(): sleep(interval) next_status = status() if next_status != current_status: logging.info(message, next_status) current_status = next_status # ZAP wrapper tool_name = "ZAP" results = list() # Start ZAP daemon in background (no need for supervisord) logging.info("Starting ZAP daemon") zap_daemon = subprocess.Popen([ "/usr/bin/java", "-Xmx499m", "-jar", "/opt/zap/zap.jar", "-daemon", "-port", "8091", "-host", "0.0.0.0", "-config", "api.key=dusty", "-config", "api.addrs.addr.regex=true", "-config", "api.addrs.addr.name=.*", "-config", "ajaxSpider.browserId=htmlunit" ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) zap_api = ZAPv2(apikey="dusty", proxies={ "http": "http://127.0.0.1:8091", "https": "http://127.0.0.1:8091" }) # Wait for zap to start zap_started = False for _ in range(600): try: logging.info("Started ZAP %s", zap_api.core.version) zap_started = True break except IOError: sleep(1) if not zap_started: logging.error("ZAP failed to start") zap_daemon.kill() zap_daemon.wait() return tool_name, results # Format target URL proto = config.get("protocol") host = config.get("host") port = config.get("port") target = f"{proto}://{host}" if (proto == "http" and int(port) != 80) or \ (proto == "https" and int(port) != 443): target = f"{target}:{port}" logging.info("Scanning target %s", target) # Setup context logging.info("Preparing context") zap_context_name = "dusty" zap_context = zap_api.context.new_context(zap_context_name) # Setup context inclusions and exclusions zap_api.context.include_in_context(zap_context_name, f".*{re.escape(host)}.*") for include_regex in config.get("include", list()): zap_api.context.include_in_context(zap_context_name, include_regex) for exclude_regex in config.get("exclude", list()): zap_api.context.exclude_from_context(zap_context_name, exclude_regex) if config.get("auth_script", None): # Load our authentication script zap_api.script.load(scriptname="zap-selenium-login.js", scripttype="authentication", scriptengine="Oracle Nashorn", filename=pkg_resources.resource_filename( "dusty", "templates/zap-selenium-login.js"), scriptdescription="Login via selenium script") # Enable use of laoded script with supplied selenium-like script zap_api.authentication.set_authentication_method( zap_context, "scriptBasedAuthentication", urllib.parse.urlencode({ "scriptName": "zap-selenium-login.js", "Script": base64.b64encode( json.dumps(config.get("auth_script")).encode( "utf-8")).decode("utf-8") })) # Add user to context zap_user = zap_api.users.new_user(zap_context, "dusty_user") zap_api.users.set_authentication_credentials( zap_context, zap_user, urllib.parse.urlencode({ "Username": config.get("auth_login", ""), "Password": config.get("auth_password", ""), "type": "UsernamePasswordAuthenticationCredentials" })) # Enable added user zap_api.users.set_user_enabled(zap_context, zap_user, True) # Setup auth indicators if config.get("logged_in_indicator", None): zap_api.authentication.set_logged_in_indicator( zap_context, config.get("logged_in_indicator")) if config.get("logged_out_indicator", None): zap_api.authentication.set_logged_out_indicator( zap_context, config.get("logged_out_indicator")) # Setup scan policy scan_policy_name = "Default Policy" scan_policies = [ item.strip() for item in config.get("scan_types", "all").split(",") ] # Disable globally blacklisted rules for item in c.ZAP_BLACKLISTED_RULES: zap_api.ascan.set_scanner_alert_threshold( id=item, alertthreshold="OFF", scanpolicyname=scan_policy_name) zap_api.pscan.set_scanner_alert_threshold(id=item, alertthreshold="OFF") if "all" not in scan_policies: # Disable all scanners first for item in zap_api.ascan.scanners(scan_policy_name): zap_api.ascan.set_scanner_alert_threshold( id=item["id"], alertthreshold="OFF", scanpolicyname=scan_policy_name) # Enable scanners from suite for policy in scan_policies: for item in c.ZAP_SCAN_POCILICES.get(policy, []): zap_api.ascan.set_scanner_alert_threshold( id=item, alertthreshold="DEFAULT", scanpolicyname=scan_policy_name) # Spider logging.info("Spidering target: %s", target) if config.get("auth_script", None): scan_id = zap_api.spider.scan_as_user(zap_context, zap_user, target, recurse=True, subtreeonly=True) else: scan_id = zap_api.spider.scan(target) _wait_for_completion(lambda: int(zap_api.spider.status(scan_id)) < 100, lambda: int(zap_api.spider.status(scan_id)), "Spidering progress: %d%%") # Wait for passive scan _wait_for_completion(lambda: int(zap_api.pscan.records_to_scan) > 0, lambda: int(zap_api.pscan.records_to_scan), "Passive scan queue: %d items") # Ajax Spider logging.info("Ajax spidering target: %s", target) if config.get("auth_script", None): scan_id = zap_api.ajaxSpider.scan_as_user(zap_context_name, "dusty_user", target, subtreeonly=True) else: scan_id = zap_api.ajaxSpider.scan(target) _wait_for_completion(lambda: zap_api.ajaxSpider.status == 'running', lambda: int(zap_api.ajaxSpider.number_of_results), "Ajax spider found: %d URLs") # Wait for passive scan _wait_for_completion(lambda: int(zap_api.pscan.records_to_scan) > 0, lambda: int(zap_api.pscan.records_to_scan), "Passive scan queue: %d items") # Active scan logging.info("Active scan against target %s", target) if config.get("auth_script", None): scan_id = zap_api.ascan.scan_as_user( target, zap_context, zap_user, recurse=True, scanpolicyname=scan_policy_name) else: scan_id = zap_api.ascan.scan(target, scanpolicyname=scan_policy_name) _wait_for_completion(lambda: int(zap_api.ascan.status(scan_id)) < 100, lambda: int(zap_api.ascan.status(scan_id)), "Active scan progress: %d%%") # Wait for passive scan _wait_for_completion(lambda: int(zap_api.pscan.records_to_scan) > 0, lambda: int(zap_api.pscan.records_to_scan), "Passive scan queue: %d items") # Get report logging.info("Scan finished. Processing results") zap_report = zap_api.core.jsonreport() if os.environ.get("debug", False): with open("/tmp/zap.json", "wb") as report_file: report_file.write(zap_report.encode("utf-8")) # Stop zap zap_daemon.kill() zap_daemon.wait() # Parse JSON results.extend(ZapJsonParser(zap_report, tool_name).items) pkg_resources.cleanup_resources() return tool_name, results
def tearDown(self): """Clean up after each test.""" pkg_resources.cleanup_resources()
def test_exit_status(): # define columns cols = { "intA" : numpy.random.randint(0,high=2**16,size=size), "intB" : numpy.random.randint(-10,high=20,size=size), "idxA" : numpy.arange(size), "chrA" : numpy.array([chr(65+(X%(91-65))) for X in range(size)]), "strA" : numpy.array([str(GenomicSegment("chrA",X,X+500,"+")) for X in range(size)]), "strB" : numpy.array([str(GenomicSegment("chrB",X/2,X/2+500,"-")) for X in range(size)]), "floatA" : 10*numpy.random.randn(size) + 500, "floatB" : (10**-5)*numpy.random.random(size), "objA" : numpy.tile(None,5000), "objB" : numpy.array([GenomicSegment("chrC",X,X+Y,"+") for X,Y in zip(range(size),numpy.random.randint(2,high=1000,size=size))]), } # allocate temp files we will use headerfile = NamedTemporaryFile(delete=False,mode="w") headerfile_extra_cols = NamedTemporaryFile(delete=False,mode="w") headerfile_extra_cols_diff = NamedTemporaryFile(delete=False,mode="w") headerfile_extra_cols_shuffled = NamedTemporaryFile(delete=False,mode="w") headerfile_shuffled = NamedTemporaryFile(delete=False,mode="w") headerfile_diff_vals = NamedTemporaryFile(delete=False,mode="w") noheaderfile = NamedTemporaryFile(delete=False,mode="w") noheaderfile_extra_cols = NamedTemporaryFile(delete=False,mode="w") noheaderfile_extra_cols_diff = NamedTemporaryFile(delete=False,mode="w") noheaderfile_extra_cols_shuffled = NamedTemporaryFile(delete=False,mode="w") noheaderfile_shuffled = NamedTemporaryFile(delete=False,mode="w") noheaderfile_diff_vals = NamedTemporaryFile(delete=False,mode="w") # write values keyorder = ["idxA"] + sorted(list(set(cols.keys()) - { "idxA" })) table1 = pd.DataFrame(cols) table1.to_csv(headerfile,index=False,header=True,sep="\t") table1.to_csv(noheaderfile,index=False,header=False,sep="\t", columns=keyorder) headerfile.close() noheaderfile.close() table1["extra"] = 2**7 * numpy.random.random(size=size) table1.to_csv(headerfile_extra_cols,index=False,header=True,sep="\t") table1.to_csv(noheaderfile_extra_cols,index=False,header=False,sep="\t", columns=["extra"]+keyorder) headerfile_extra_cols.close() noheaderfile_extra_cols.close() table1["extra"] += 10**-4 * numpy.random.random(size=size) table1.to_csv(headerfile_extra_cols_diff,index=False,header=True,sep="\t") table1.to_csv(noheaderfile_extra_cols_diff,index=False,header=False,sep="\t", columns=["extra"]+keyorder) headerfile_extra_cols_diff.close() noheaderfile_extra_cols_diff.close() shufidx = numpy.arange(size) shuffle(shufidx) table2 = pd.DataFrame({ K : V[shufidx] for K,V in cols.items()}) table2.to_csv(headerfile_shuffled,index=False,header=True,sep="\t") table2.to_csv(noheaderfile_shuffled,index=False,header=False,sep="\t", columns=keyorder) headerfile_shuffled.close() noheaderfile_shuffled.close() table2["extra"] = table1["extra"][shufidx] table2.to_csv(headerfile_extra_cols_shuffled,index=False,header=True,sep="\t") table2.to_csv(noheaderfile_extra_cols_shuffled, index=False,header=False,sep="\t", columns=["extra"]+keyorder) headerfile_extra_cols_shuffled.close() noheaderfile_extra_cols_shuffled.close() # Define tests, as tuples of: # -Test name/description # -Command-line arguments to pass to :py:mod:`plastid.bin.test_table_equality` # -Expected exit code/returns status for :py:func:`main` tests = [ ("same", "%s %s" % (headerfile.name,headerfile.name), 0), ("diff_column_names", "%s %s" % (headerfile.name,headerfile_extra_cols.name), 1), ("extra_column_names_ignored", "%s %s --exclude extra" % (headerfile.name,headerfile_extra_cols.name), 0), ("shuffled_rows", "%s %s" % (headerfile.name,headerfile_shuffled.name), 1), ("shuffled_rows_name_sort", "%s %s --sort_keys idxA" % (headerfile.name,headerfile_shuffled.name), 0), ("shuffled_rows_multi_name_sort", "%s %s --sort_keys strB chrA" % (headerfile.name,headerfile_shuffled.name), 0), ("same_column_names_diff_values", "%s %s" % (headerfile_extra_cols.name,headerfile_extra_cols_diff.name), 1), ("same_column_names_diff_values_tol", "%s %s --tol 0.01" % (headerfile_extra_cols.name,headerfile_extra_cols_diff.name), 0), ("same_column_names_diff_values_ignored", "%s %s --exclude extra" % (headerfile_extra_cols.name,headerfile_extra_cols_diff.name), 0), ("shuffled_rows_extra_columns_ignored", "%s %s --exclude extra" % (headerfile.name,headerfile_extra_cols_shuffled.name), 1), ("shuffled_rows_extra_columns_ignored_name_sort", "%s %s --exclude extra --sort_keys idxA" % (headerfile.name,headerfile_extra_cols_shuffled.name), 0), ("noheader_same", "%s %s --no_header" % (noheaderfile.name,noheaderfile.name), 0), ("noheader_extra_columns", "%s %s --no_header" % (noheaderfile.name,noheaderfile_extra_cols.name), 1), ("noheader_shuffled_rows", "%s %s --no_header" % (noheaderfile.name,noheaderfile_shuffled.name), 1), ("noheader_shuffled_rows_int_sort", "%s %s --no_header --sort_keys 0" % (noheaderfile.name,noheaderfile_shuffled.name), 0), ("noheader_diff_values", "%s %s --no_header" % (noheaderfile_extra_cols.name,noheaderfile_extra_cols_diff.name), 1), ("noheader_diff_values_tol", "%s %s --no_header --tol 0.01" % (noheaderfile_extra_cols.name,noheaderfile_extra_cols_diff.name), 0), ("no_header_diff_values_ignored", "%s %s --no_header --exclude 0" % (noheaderfile_extra_cols.name,noheaderfile_extra_cols_diff.name), 0), ("no_header_shuffled_rows_extra_columns_ignored_int_sort", "%s %s --no_header --exclude 0 --sort_keys 1" % (noheaderfile_extra_cols.name,noheaderfile_extra_cols_shuffled.name), 0) ] """ Tests to conduct, as tuples of: - Test name/description - Command-line arguments to pass to :py:mod:`plastid.bin.test_table_equality` - Expected exit code/returns status for :py:func:`main` """ for test_name, argstr, expected_exit in tests: yield check_exit_status, test_name, argstr, expected_exit # clean up os.unlink(headerfile.name ) os.unlink(headerfile_extra_cols.name ) os.unlink(headerfile_extra_cols_diff.name ) os.unlink(headerfile_extra_cols_shuffled.name ) os.unlink(headerfile_shuffled.name ) os.unlink(headerfile_diff_vals.name ) os.unlink(noheaderfile.name ) os.unlink(noheaderfile_extra_cols.name ) os.unlink(noheaderfile_extra_cols_diff.name ) os.unlink(noheaderfile_extra_cols_shuffled.name ) os.unlink(noheaderfile_shuffled.name ) os.unlink(noheaderfile_diff_vals.name ) cleanup_resources()
def main(): import warnings warnings.filterwarnings('ignore') ap = argparse.ArgumentParser() ap.add_argument("infile", type=str, help="file to process") ap.add_argument("-f", dest="rules", default= resource_filename(__name__, RULESFILE), help="name of file containing transformation rules" + " (default: {})".format(RULESFILE)) ap.add_argument("-o", dest="outfile", default=OUTFILE, help="destination file" + " (default: {})".format(OUTFILE)) ap.add_argument("-m", dest="marker", default="", help="marker to be added to new nodes and edges" + " (default: None)") args = ap.parse_args() with open(args.outfile, "w") as outfile: # write declaration # write open corpus tag print('<?xml version="1.0"', 'encoding="ISO-8859-1"', 'standalone="yes"?>', file=outfile) print('<corpus', 'id="Release2_1-{:%y%m%d}">'.format(date.today()), file=outfile) _, head = next(etree.iterparse(args.infile, tag="head", remove_blank_text=True)) # if marker requested # insert test nodes into features section in header if args.marker: new_nodes_list = new_node_types(args.rules, args.marker) cat_features = head.find("annotation/feature[@name='cat']") add_new_types(cat_features, new_nodes_list) new_edges_list = new_edge_types(args.rules, args.marker) edge_features = head.find("annotation/edgelabel") add_new_types(edge_features, new_edges_list) secedge_features = head.find("annotation/secedgelabel") add_new_types(secedge_features, new_edges_list) else: new_nodes_list = None new_edges_list = None # write head data print(str(etree.tostring(head, pretty_print=True), "utf-8"), file=outfile) # get all rules all_rules = list(load_rules(args.rules, args.infile, new_nodes=new_nodes_list, new_edges=new_edges_list, _marker=args.marker)) # write open body tag print("<body>", file=outfile) # iterate through all sentences # based on fast_iter from: # http://www.ibm.com/developerworks/xml/library/x-hiperfparse/ for _, element in etree.iterparse(args.infile, tag=SENTENCE_TAG, remove_blank_text=True): sent = process_sentence(element, all_rules) print(str(etree.tostring(sent, pretty_print=True), "iso-8859-1"), file=outfile) element.clear() while element.getprevious() is not None: del element.getparent()[0] # write close body tag and close corpus tag print("</body>\n</corpus>", file=outfile) print("\ndone!") cleanup_resources() exit()
""" Utility functions """ import rastercube import numpy as np import os import errno from datetime import datetime import calendar import cPickle as pickle import pkg_resources import atexit # Cleanup tmpdir used by asset_fname on interpreter exit atexit.register(lambda: pkg_resources.cleanup_resources()) def asset_fname(relpath): """ Gets the filename to an asset relative to the rastercube package root. When rastercube is packaged as an egg, you can't access assets using os.path.join(rastercube.__file__, 'assets/foo.json') since the egg is a zip. So you should use this function. See : http://peak.telecommunity.com/DevCenter/PythonEggs#accessing-package-resources >>> fname = asset_fname('assets/modis_tiles.geojson') """ return pkg_resources.resource_filename(rastercube.__name__, relpath)