def search(self,search_str,search_dir,depth=2): # Depth -> number of top nodes to search. Default 2 (arbitrary) has been sufficient so far. results = Counter() # {url:score} dist_neu_searchq=Counter() # {nid:dist} Contains the similarity of each neuron to the aggregated search query neuron_lookup = self._neu_index neuron_labels = [[k for (k,n) in Counter(neuron.get_keylist()).most_common()[:10]] for neuron in self._neurons] glomat = self._glomat # Global matrix. Contains similarity of each search term in the query to all neurons. Something like a cache. conn = SQLCon() searchvecs = [(x,list(conn.read(x))) for x in search_str.split() if not conn.read(x) is None] # Obtain (word,vec) of search terms search_len = len(searchvecs) for (w,v) in searchvecs: # For colour coding the map try: for nid in glomat[w]: if glomat[w][nid] > dist_neu_searchq[nid]: dist_neu_searchq[nid] += glomat[w][nid]/search_len except KeyError: glomat[w]={} for nid,neuron in enumerate(self._neurons): glomat[w][nid] = distance(neuron.get_weights(),v) # cosine similarity, hence 1 is best. 0 is bleh. -1 is opposite. if glomat[w][nid] > dist_neu_searchq[nid]: dist_neu_searchq[nid] += glomat[w][nid]/search_len # Union of all doclists with minimum dist_from_neuron. doclist = {} for nid in dist_neu_searchq.most_common()[:depth]: neuron = neuron_lookup[nid[0]] doclist.update(neuron.get_top_docs(30)) files = (open(doc) for doc in doclist) for json_file in files: data = json.load(json_file) centroids = data['centroids'] url = data['url'] json_file.close() wc_sim = [distance(v,c) for c in centroids] max_wc_sim = max(wc_sim) results[url] += max_wc_sim/len(searchvecs) results = OrderedDict(results.most_common(20)) htmlVars = {'query': search_str, 'results':results} htmlCode = template.render(htmlVars) result_path = os.path.join(search_dir,search_str+'.html') map_path = os.path.join(search_dir,search_str+'_map.html') with open(result_path,'w') as f: f.write(htmlCode) self.draw_SOM(search_str,dist_neu_searchq,neuron_labels,map_path) result_path = "file://{}".format(pathname2url(result_path)) map_path = "file://{}".format(pathname2url(map_path)) webbrowser.open(result_path)
def repl_relative(m, base_path, relative_path): """ Replace path with relative path """ RE_WIN_DRIVE_PATH = re.compile(r"(^(?P<drive>[A-Za-z]{1}):(?:\\|/))") link = m.group(0) try: scheme, netloc, path, params, query, fragment, is_url, is_absolute = parse_url(m.group('path')[1:-1]) if not is_url: # Get the absolute path of the file or return # if we can't resolve the path path = url2pathname(path) abs_path = None if (not is_absolute): # Convert current relative path to absolute temp = os.path.normpath(os.path.join(base_path, path)) if os.path.exists(temp): abs_path = temp.replace("\\", "/") elif os.path.exists(path): abs_path = path if abs_path is not None: convert = False # Determine if we should convert the relative path # (or see if we can realistically convert the path) if (sublime.platform() == "windows"): # Make sure basepath starts with same drive location as target # If they don't match, we will stay with absolute path. if (base_path.startswith('//') and base_path.startswith('//')): convert = True else: base_drive = RE_WIN_DRIVE_PATH.match(base_path) path_drive = RE_WIN_DRIVE_PATH.match(abs_path) if ( (base_drive and path_drive) and base_drive.group('drive').lower() == path_drive.group('drive').lower() ): convert = True else: # OSX and Linux convert = True # Convert the path, url encode it, and format it as a link if convert: path = pathname2url(os.path.relpath(abs_path, relative_path).replace('\\', '/')) else: path = pathname2url(abs_path) link = '%s"%s"' % (m.group('name'), urlunparse((scheme, netloc, path, params, query, fragment))) except: # Parsing crashed an burned; no need to continue. pass return link
def do_GET(self): if self.path in map( lambda x: urllib.pathname2url(os.path.join('/', x)), self.server.allowed_basenames ): utils.logger.info(_("Peer found. Uploading...")) full_path = os.path.join(os.curdir, self.server.filename) with open(full_path, 'rb') as fh: maxsize = os.path.getsize(full_path) self.send_response(200) self.send_header('Content-type', 'application/octet-stream') self.send_header( 'Content-disposition', 'inline; filename="%s"' % os.path.basename( self.server.filename ) ) self.send_header('Content-length', maxsize) self.end_headers() i = 0 while True: data = fh.read(1024 * 8) # chunksize taken from urllib if not data: break self.wfile.write(data) if self.server.reporthook is not None: self.server.reporthook(i, 1024 * 8, maxsize) i += 1 self.server.downloaded = True else: self.send_response(404) self.end_headers() raise RuntimeError(_("Invalid request received. Aborting."))
def _create_tar_overlay(self): repo_name = 'tar-test-overlay' tar_source_path = os.path.join(HERE, 'testfiles', 'layman-test.tar.bz2') # Duplicate test tarball (so we can delete it after testing) (_, temp_tarball_path) = tempfile.mkstemp() shutil.copyfile(tar_source_path, temp_tarball_path) # Write overlay collection XML xml_text = '''\ <?xml version="1.0" encoding="UTF-8"?> <repositories xmlns="" version="1.0"> <repo quality="experimental" status="unofficial"> <name>%(repo_name)s</name> <description>XXXXXXXXXXX</description> <owner> <email>[email protected]</email> </owner> <source type="tar">file://%(temp_tarball_url)s</source> </repo> </repositories> '''\ % { 'temp_tarball_url': urllib.pathname2url(temp_tarball_path), 'repo_name': repo_name } print(xml_text) return xml_text, repo_name, temp_tarball_path
def h5toncml(h5fname): """Generate NcML representation of HDF5 file's content. Dataset values not included. :arg str h5fname: HDF5 file name. :return: An instance of lxml.etree.ElementTree representing the NcML content. """ f = h5py.File(h5fname, 'r') # Create the root element and the document... root = etree.Element(etree.QName(ns['nc'], 'netcdf'), nsmap=ns) ncmldoc = etree.ElementTree(root) # Add the XML schema attributes... root.attrib[etree.QName(ns['xsi'], 'schemaLocation')] = \ 'http://www.unidata.ucar.edu/schemas/netcdf/ncml-2.2.xsd' # Add location attribute... root.attrib['location'] = urlparse.urljoin( 'file:', urlrequest.pathname2url(osp.abspath(h5fname))) # Visit each HDF5 object... grp_node['/'] = root do_attributes(root, f) f.visititems(objinfo) f.close() return ncmldoc
def __init__(self, **kwargs): """Creates a lab notebook Entry object instance. Parses the relevant file corresponding to the notebook entry and attempts to determine an appropriate title to use for the entry, the date the analysis was last modified, etc. and stores all of the relevant information as a Entry instance. Args: filepath: Path to the file for which the lab notebook entry is being created. output_dir: The notebook HTML output directory. This will be removed from the final path in order to generate a relative URL. url_prefix: An optional URL prefix to be preprended to the entry. """ self.filepath = kwargs['filepath'] self.filename = os.path.basename(self.filepath) self.dir_name = os.path.basename(os.path.dirname(self.filepath)) self.date = datetime.fromtimestamp(os.path.getmtime(self.filepath)) self.url = pathname2url(urljoin(kwargs['url_prefix'], self.filepath.replace(kwargs['output_dir'] + "/", ''))) # set title if 'title' in kwargs: self.title = kwargs['title'] else: self.title = self._get_entry_title()
def test_branch_no_commit(self): """Test branching when no commits are involved""" repo = self.make_repo(( dict(nodes=( dict(action="add", path="trunk", kind="dir"), dict(action="add", path="branches", kind="dir"), dict(action="add", path="trunk/file", kind="file", content=b""), )), dict(nodes=( dict(action="add", path="branches/branch", copyfrom_path="trunk", copyfrom_rev=1), )), )) url = "file://{}/branches/branch".format(pathname2url(repo)) output = os.path.join(self.dir, "output") with svnex.FastExportFile(output) as fex: rev_map = {"trunk": {1: "trunk"}} exporter = svnex.Exporter(url, fex, root="", rev_map=rev_map, quiet=True) exporter.export("refs/heads/branch") with open(output, "r", encoding="ascii") as output: self.assertMultiLineEqual("""\ reset refs/heads/branch from trunk """, output.read())
def test_less_fields(self): filename = os.path.join(os.path.dirname(__file__), 'resources/CSV_TEST_DATA_EMPTY_FIELDS.csv') mediator = CsvMediator(first_row_is_headers=True, restval="Worcester") uri = urljoin('file:', urllib.pathname2url(filename)) data = mediator.load(uri=uri, base_name="test", read_on_load=True) self.assertEquals(len(data), 1) # Only one type of elements row_class = list(data.keys())[0] self.assertEquals(len(data[row_class]), 25) # Ten rows row_element = data[row_class][0] self.assertIsInstance(row_element, presentation.Row) self.assertTrue(hasattr(row_element, "id")) self.assertTrue(hasattr(row_element, "first_name")) self.assertTrue(hasattr(row_element, "last_name")) self.assertTrue(hasattr(row_element, "email")) self.assertTrue(hasattr(row_element, "country")) self.assertTrue(hasattr(row_element, "city")) self.assertEqual(row_element.id, "1") self.assertEqual(row_element.first_name, "Chris") self.assertEqual(row_element.last_name, "Jordan") self.assertEqual(row_element.email, "*****@*****.**") self.assertEqual(row_element.country, "Indonesia") self.assertEqual(row_element.city, "Worcester") row_element = data[row_class][1] self.assertEqual(row_element.id, "2") self.assertEqual(row_element.first_name, "Edward") self.assertEqual(row_element.last_name, "Williamson") self.assertEqual(row_element.email, "*****@*****.**") self.assertEqual(row_element.country, "Brazil") self.assertEqual(row_element.city, "Vila Velha")
def _open_file( self, filename ): uri = self._rootdir + "/" + pathname2url(filename) gio_file = Gio.file_new_for_uri(uri) tab = self._window.get_tab_from_location(gio_file) if tab == None: tab = self._window.create_tab_from_location( gio_file, None, 0, 0, False, False ) self._window.set_active_tab( tab )
def path_to_url(path): """Convert a system path to a URL.""" if os.path.sep == '/': return path return pathname2url(path)
def build_resources(self): resources = [] if not self.root_dir: return resources for root, dirs, files in os.walk(self.root_dir, followlinks=True): for file_name in files: path = os.path.join(root, file_name) if os.path.getsize(path) > MAX_FILESIZE_BYTES: continue with open(path, 'rb') as f: content = f.read() path_for_url = pathname2url(path.replace(self.root_dir, '', 1)) if self.base_url[-1] == '/' and path_for_url[0] == '/': path_for_url = path_for_url.replace('/', '' , 1) resource_url = "{0}{1}".format(self.base_url, path_for_url) resource = percy.Resource( resource_url=resource_url, sha=utils.sha256hash(content), local_path=os.path.abspath(path), ) resources.append(resource) return resources
def _build_app(self): # build window w = Gtk.Window() w.set_position(Gtk.WindowPosition.CENTER) w.set_wmclass('Welcome to Linux Lite', 'Welcome to Linux Lite') w.set_title('Welcome to Linux Lite') w.set_size_request(768, 496) w.set_icon_from_file(os.path.join(self._data_path, "icons/lite-welcome.png")) # build webkit container mv = LiteAppView() # load our index file file_out = os.path.abspath(os.path.join(self._data_path, 'frontend/index.html')) uri = 'file://' + pathname2url(file_out) mv.open(uri) # build scrolled window widget and add our appview container sw = Gtk.ScrolledWindow() sw.set_policy(Gtk.PolicyType.NEVER, Gtk.PolicyType.AUTOMATIC) sw.add(mv) # build a an autoexpanding box and add our scrolled window b = Gtk.VBox(homogeneous=False, spacing=0) b.pack_start(sw, expand=True, fill=True, padding=0) # add the box to the parent window and show w.add(b) w.connect('delete-event', self.close) w.show_all() self._window = w self._appView = mv
def share(filename, forever): """Share a file in the local network.""" ip = utils.get_ip() # port = get_port() # Bind to port 0. OS assigns a random open port. server = httpserver.HTTPServer((ip, 0), utils.LocalFileHandler) port = server.server_port server.filename = filename zc_info = zeroconf.ServiceInfo( "_http._tcp.local.", "%s._http._tcp.local." % filename, utils.ip_to_bytes(ip), port, 0, 0, {'filename': filename} ) url = "http://" + ip + ":" + str(port) + "/" + urllib.pathname2url(filename) zc_instance = zeroconf.Zeroconf() try: zc_instance.register_service(zc_info) click.echo('Sharing %s at %s' % (filename, url)) if forever: server.serve_forever(poll_interval=0.5) else: server.handle_request() click.echo('File downloaded by peer. Exiting') sys.exit(0) except KeyboardInterrupt: pass
def map_files(): """List all available files.""" files = OrderedDict() zc_instance = zeroconf.Zeroconf() listener = utils.ServiceListener() zeroconf.ServiceBrowser(zc_instance, "_http._tcp.local.", listener) try: # Give listener some time to discover available services. time.sleep(0.5) if not listener.services: click.echo('No files available. Waiting ...') while not listener.services: time.sleep(0.5) click.echo('Peer(s) found.') for service in listener.services: address = utils.bytes_to_ip(service.address) port = service.port filename = service.properties[b'filename'].decode('utf-8') url = "http://" + address + ":" + str(port) + "/" + \ urllib.pathname2url(filename) files[filename] = url except KeyboardInterrupt: sys.exit(0) return files
def _fetch_data(self, path, size): self._curl.setopt(pycurl.URL, "ftp://{}{}".format(self._server_addr, pathname2url(path))) try: tagsize = self._get_tag_size() if tagsize is not None: # Tags are at the beginning of the file, so download some more bytes self._fetch_range("0-%d" % (tagsize + 10)) else: # Tags are perhaps at the end of file then self._fetch_range("%d-%d" % (size - 10, size)) tagsize = self._get_tag_size() if tagsize is not None: self._fetch_range("%d-%d" % (size - tagsize, size - 10)) else: # See if there is ID3v1 self._fetch_range("%d-%d" % (size - 128, size)) if self._buffer.getvalue()[0:3] == b"TAG": id3v1 = self._buffer.getvalue() self._fetch_range("%d-%d" % (size - 138, size - 129)) tagsize = self._get_tag_size() # See if there is ID3v2 before ID3v1 if tagsize is not None: self._fetch_range("%d-%d" % (size - tagsize - 148, size - 129)) self._buffer.write(id3v1) # Concat ID3v1 to ID3v2 self._buffer.seek(0) return True except pycurl.error as e: self.log.exception("_fetch_data error", e) return False
def gstDuration(path): try: disc = GstPbutils.Discoverer() discInfo = disc.discover_uri('file://' + url.pathname2url(path)) return int(discInfo.get_duration() / Gst.MSECOND) except: return -1
def _create_layout(self): box = gui.widgetBox(self.controlArea, orientation='horizontal') self.varmodel = VariableListModel(parent=self) self.attr_combo = gui.comboBox(box, self, 'selected_attr', orientation=Qt.Horizontal, label='Region attribute:', callback=self.on_attr_change, sendSelectedValue=True) self.attr_combo.setModel(self.varmodel) self.map_combo = gui.comboBox(box, self, 'selected_map', orientation=Qt.Horizontal, label='Map type:', callback=self.on_map_change, items=Map.all) hexpand = QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) self.attr_combo.setSizePolicy(hexpand) self.map_combo.setSizePolicy(hexpand) url = urljoin('file:', pathname2url(os.path.join( os.path.dirname(__file__), 'resources', 'owgeomap.html'))) self.webview = gui.WebviewWidget(self.controlArea, self, url=QUrl(url)) self.controlArea.layout().addWidget(self.webview) QTimer.singleShot( 0, lambda: self.webview.evalJS('REGIONS = {};'.format({Map.WORLD: CC_WORLD, Map.EUROPE: CC_EUROPE, Map.USA: CC_USA})))
def get_file_url(path: str) -> str: """Return the file URL that corresponds to the file path ``path``. If ``path`` is relative, it is converted into an absolute path before being converted into a file URL. """ return _urlparse.urljoin('file:', _urlreq.pathname2url(_os.path.abspath(path)))
def _get_box_from_loc_name(self, loc_name): loc_name = pathname2url(loc_name) api_url = 'https://maps.googleapis.com/maps/api/geocode/json?address={}' api_url = api_url.format(loc_name) req = urlopen(api_url) body = ''.join(map(lambda x: x.decode('utf-8'), req.readlines())) result = json.loads(body)['results'][0] geo_box = result['geometry']['bounds'] box = [geo_box['southwest']['lng'], geo_box['southwest']['lat'], geo_box['northeast']['lng'], geo_box['northeast']['lat']] if self.expand: mid = [(box[2] - box[0]) / 2 + box[0], (box[3] - box[1]) / 2 + box[1]] var_lng, var_lat = self._meter_to_geo(mid[0], mid[1], self.expand) box[0] -= var_lng box[1] -= var_lat box[2] += var_lng box[3] += var_lat return box
def test_modify_branch(self): """Modification of branch directory properties""" repo = self.make_repo(( dict(nodes=(dict(action="add", path="trunk", kind="dir"),)), dict(nodes=( dict(action="change", path="trunk", props={"name": "value"}), )), )) url = "file://{}/trunk".format(pathname2url(repo)) output = os.path.join(self.dir, "output") with svnex.FastExportFile(output) as fex: exporter = svnex.Exporter(url, fex, root="", quiet=True) exporter.export("refs/ref") with open(output, "r", encoding="ascii") as output: self.assertMultiLineEqual("""\ commit refs/ref mark :1 committer (no author) <(no author)@00000000-0000-0000-0000-000000000000> 0 +0000 data 60 git-svn-id: /trunk@1 00000000-0000-0000-0000-000000000000 commit refs/ref mark :2 committer (no author) <(no author)@00000000-0000-0000-0000-000000000000> 0 +0000 data 60 git-svn-id: /trunk@2 00000000-0000-0000-0000-000000000000 """, output.read())
def make_repository(self, relpath, allow_revprop_changes=True): """Create a repository. :return: Handle to the repository. """ abspath = os.path.join(self.test_dir, relpath) repos.create(abspath) if allow_revprop_changes: if sys.platform == "win32": revprop_hook = os.path.join(abspath, "hooks", "pre-revprop-change.bat") f = open(revprop_hook, "w") try: f.write("exit 0\n") finally: f.close() else: revprop_hook = os.path.join(abspath, "hooks", "pre-revprop-change") f = open(revprop_hook, "w") try: f.write("#!/bin/sh\n") finally: f.close() os.chmod(revprop_hook, os.stat(revprop_hook).st_mode | 0o111) if sys.platform == "win32": return "file:%s" % pathname2url(abspath) else: return "file://%s" % abspath
def path2url(path): m = re.match(r'(.*)[/\\]index.html?$', path) if m: path = m.group(1) + os.path.sep path = os.path.sep + path url = pathname2url(path.encode(URL_ENCODING)) return url.decode('ASCII') if PY2 else url
def _get_uri(self): uri = self.filename if not uri: return if uri.split(":")[0] not in ("http", "https", "file", "udp", "rtp", "rtsp"): uri = "file:" + pathname2url(path.realpath(uri)) return uri
def load_window(): controller = ColorsController() cur_dir = os.path.dirname(os.path.abspath(__file__)) file_ = os.path.join(cur_dir, 'html/index.html') uri = 'file://' + pathname2url(file_) zaguan = Zaguan(uri, controller) zaguan.run(debug=True)
def ffmpeg_take_scene_screenshot_without_save(scene): # When we get here we assume that the scene was alrady probed with ffprobe, therefore it has all the video metadata. # {10 /x + y = 1} , {7200 / x + y = 300} where 10 and 7200 are the duration of the original video # and 1 and 300 are the time in seconds where we should take the screen shot. # EX if the video is 45 second long the screenshot will be taken on the 2nd second # if the video duration is half an hour the screenshot will be taken at 74 seconds # x=24.0468 y=0.584145 x = 24.0468 y = 0.584145 screenshot_time = seconds_to_string(int(scene.duration / x + y)) a = ffmpeg_take_screenshot(screenshot_time, scene.path_to_file) if a['success']: # print("Screenshot Taken") dest_path = os.path.join(MEDIA_PATH, "scenes", str(scene.id), "thumb") z = move_sample_movie_to_correct_dir(scene, True, "thumb.jpg", dest_path, SCREENSHOT_OUTPUT_PATH, 'image') time.sleep(1) thumb_path = os.path.relpath(z, start='videos') as_uri = urllib.pathname2url(thumb_path) scene.thumbnail = as_uri
def getFileCopyState(filename): global fileCopyState statestring = urlparse.urljoin('file:', urllib.pathname2url(filename)) #print("statestring {0}".format(statestring)) if statestring in fileCopyState: return fileCopyState[statestring] else: return -1
def test_no_read_on_load(self): filename = os.path.join(os.path.dirname(__file__), 'resources/CSV_TEST_DATA.csv') mediator = CsvMediator(first_row_is_headers=True) uri = urljoin('file:', urllib.pathname2url(filename)) data = mediator.load(uri=uri, base_name="test", read_on_load=False) self.assertEquals(len(data), 1) # Only one type of elements for k, v in data.items(): self.assertEquals(len(v), 0)
def path2url(url): """URL to path.""" path = pathname2url(url) # If on windows, replace the notation to use a default protocol `///` with nothing. if is_win() and RE_WIN_DEFAULT_PROTOCOL.match(path): path = path.replace('///', '', 1) return path
def test_row_class_is_subclass_of_presentation_row(self): filename = os.path.join(os.path.dirname(__file__), 'resources/CSV_TEST_DATA.csv') mediator = CsvMediator(first_row_is_headers=True) uri = urljoin('file:', urllib.pathname2url(filename)) data = mediator.load(uri=uri, base_name="test", read_on_load=True) self.assertEquals(len(data), 1) # Only one type of elements row_class = list(data.keys())[0] self.assertTrue(issubclass(row_class, presentation.Row))
def path_to_url(path): """ Convert a path to a file: URL. The path will be made absolute and have quoted path parts. """ path = os.path.normpath(os.path.abspath(path)) url = urlparse.urljoin("file:", urllib2.pathname2url(path)) return url
def get_subtask(cmd_action, file_dep=None): """Return a dictionary defining a substack for string 'cmd_action'.""" if cmd_action.startswith("poetry run "): name = cmd_action.split(" ")[2] else: name = cmd_action.split(" ")[0] task = {"name": name, "actions": [cmd_action], "task_dep": ["install"]} if file_dep is not None: task["file_dep"] = file_dep return task def open_in_browser(file_to_open): """Open a file in the web browser.""" url = "file://" + pathname2url(os.path.abspath(file_to_open)) webbrowser.open(url) def targets_exists(task): """Return True (updated) if all task targets exists.""" return all([os.path.exists(target) for target in task.targets]) def get_stdout(command): """Run command with text capture and check, then return stdout.""" return check_output(command, universal_newlines=True) @contextmanager def checkout(branch):
def run(self): """Executes the command line module, taking the system arguments, determining the plugin to run and then running it.""" volatility3.framework.require_interface_version(1, 0, 0) renderers = dict([ (x.name.lower(), x) for x in framework.class_subclasses(text_renderer.CLIRenderer) ]) parser = volargparse.HelpfulArgParser( add_help=False, prog=self.CLI_NAME, description="An open-source memory forensics framework") parser.add_argument( "-h", "--help", action="help", default=argparse.SUPPRESS, help= "Show this help message and exit, for specific plugin options use '{} <pluginname> --help'" .format(parser.prog)) parser.add_argument("-c", "--config", help="Load the configuration from a json file", default=None, type=str) parser.add_argument( "--parallelism", help="Enables parallelism (defaults to off if no argument given)", nargs='?', choices=['processes', 'threads', 'off'], const='processes', default=None, type=str) parser.add_argument( "-e", "--extend", help="Extend the configuration with a new (or changed) setting", default=None, action='append') parser.add_argument( "-p", "--plugin-dirs", help="Semi-colon separated list of paths to find plugins", default="", type=str) parser.add_argument( "-s", "--symbol-dirs", help="Semi-colon separated list of paths to find symbols", default="", type=str) parser.add_argument("-v", "--verbosity", help="Increase output verbosity", default=0, action="count") parser.add_argument("-l", "--log", help="Log output to a file as well as the console", default=None, type=str) parser.add_argument( "-o", "--output-dir", help="Directory in which to output any generated files", default=os.getcwd(), type=str) parser.add_argument("-q", "--quiet", help="Remove progress feedback", default=False, action='store_true') parser.add_argument( "-r", "--renderer", metavar='RENDERER', help="Determines how to render the output ({})".format(", ".join( list(renderers))), default="quick", choices=list(renderers)) parser.add_argument( "-f", "--file", metavar='FILE', default=None, type=str, help= "Shorthand for --single-location=file:// if single-location is not defined" ) parser.add_argument( "--write-config", help="Write configuration JSON file out to config.json", default=False, action='store_true') parser.add_argument("--clear-cache", help="Clears out all short-term cached items", default=False, action='store_true') # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. known_args = [ arg for arg in sys.argv if arg != '--help' and arg != '-h' ] partial_args, _ = parser.parse_known_args(known_args) banner_output = sys.stdout if renderers[partial_args.renderer].structured_output: banner_output = sys.stderr banner_output.write("Volatility 3 Framework {}\n".format( constants.PACKAGE_VERSION)) if partial_args.plugin_dirs: volatility3.plugins.__path__ = [ os.path.abspath(p) for p in partial_args.plugin_dirs.split(";") ] + constants.PLUGINS_PATH if partial_args.symbol_dirs: volatility3.symbols.__path__ = [ os.path.abspath(p) for p in partial_args.symbol_dirs.split(";") ] + constants.SYMBOL_BASEPATHS if partial_args.log: file_logger = logging.FileHandler(partial_args.log) file_logger.setLevel(1) file_formatter = logging.Formatter( datefmt='%y-%m-%d %H:%M:%S', fmt='%(asctime)s %(name)-12s %(levelname)-8s %(message)s') file_logger.setFormatter(file_formatter) vollog.addHandler(file_logger) vollog.info("Logging started") if partial_args.verbosity < 3: if partial_args.verbosity < 1: sys.tracebacklimit = None console.setLevel(30 - (partial_args.verbosity * 10)) else: console.setLevel(10 - (partial_args.verbosity - 2)) vollog.info("Volatility plugins path: {}".format( volatility3.plugins.__path__)) vollog.info("Volatility symbols path: {}".format( volatility3.symbols.__path__)) # Set the PARALLELISM if partial_args.parallelism == 'processes': constants.PARALLELISM = constants.Parallelism.Multiprocessing elif partial_args.parallelism == 'threads': constants.PARALLELISM = constants.Parallelism.Threading else: constants.PARALLELISM = constants.Parallelism.Off if partial_args.clear_cache: framework.clear_cache() # Do the initialization ctx = contexts.Context() # Construct a blank context failures = framework.import_files( volatility3.plugins, True) # Will not log as console's default level is WARNING if failures: parser.epilog = "The following plugins could not be loaded (use -vv to see why): " + \ ", ".join(sorted(failures)) vollog.info(parser.epilog) automagics = automagic.available(ctx) plugin_list = framework.list_plugins() seen_automagics = set() chosen_configurables_list = {} for amagic in automagics: if amagic in seen_automagics: continue seen_automagics.add(amagic) if isinstance(amagic, interfaces.configuration.ConfigurableInterface): self.populate_requirements_argparse(parser, amagic.__class__) subparser = parser.add_subparsers( title="Plugins", dest="plugin", description="For plugin specific options, run '{} <plugin> --help'" .format(self.CLI_NAME), action=volargparse.HelpfulSubparserAction) for plugin in sorted(plugin_list): plugin_parser = subparser.add_parser( plugin, help=plugin_list[plugin].__doc__) self.populate_requirements_argparse(plugin_parser, plugin_list[plugin]) ### # PASS TO UI ### # Hand the plugin requirements over to the CLI (us) and let it construct the config tree # Run the argparser args = parser.parse_args() if args.plugin is None: parser.error("Please select a plugin to run") vollog.log(constants.LOGLEVEL_VVV, "Cache directory used: {}".format(constants.CACHE_PATH)) plugin = plugin_list[args.plugin] chosen_configurables_list[args.plugin] = plugin base_config_path = "plugins" plugin_config_path = interfaces.configuration.path_join( base_config_path, plugin.__name__) # Special case the -f argument because people use is so frequently # It has to go here so it can be overridden by single-location if it's defined # NOTE: This will *BREAK* if LayerStacker, or the automagic configuration system, changes at all ### if args.file: file_name = os.path.abspath(args.file) if not os.path.exists(file_name): vollog.log(logging.INFO, "File does not exist: {}".format(file_name)) else: single_location = "file:" + request.pathname2url(file_name) ctx.config[ 'automagic.LayerStacker.single_location'] = single_location # UI fills in the config, here we load it from the config file and do it before we process the CL parameters if args.config: with open(args.config, "r") as f: json_val = json.load(f) ctx.config.splice( plugin_config_path, interfaces.configuration.HierarchicalDict(json_val)) # It should be up to the UI to determine which automagics to run, so this is before BACK TO THE FRAMEWORK automagics = automagic.choose_automagic(automagics, plugin) for amagic in automagics: chosen_configurables_list[amagic.__class__.__name__] = amagic if ctx.config.get('automagic.LayerStacker.stackers', None) is None: ctx.config[ 'automagic.LayerStacker.stackers'] = stacker.choose_os_stackers( plugin) self.output_dir = args.output_dir if not os.path.exists(self.output_dir): parser.error( "The output directory specified does not exist: {}".format( self.output_dir)) self.populate_config(ctx, chosen_configurables_list, args, plugin_config_path) if args.extend: for extension in args.extend: if '=' not in extension: raise ValueError( "Invalid extension (extensions must be of the format \"conf.path.value='value'\")" ) address, value = extension[:extension.find('=')], json.loads( extension[extension.find('=') + 1:]) ctx.config[address] = value ### # BACK TO THE FRAMEWORK ### constructed = None try: progress_callback = PrintedProgress() if args.quiet: progress_callback = MuteProgress() constructed = plugins.construct_plugin( ctx, automagics, plugin, base_config_path, progress_callback, self.file_handler_class_factory()) if args.write_config: vollog.debug("Writing out configuration data to config.json") with open("config.json", "w") as f: json.dump(dict(constructed.build_configuration()), f, sort_keys=True, indent=2) except exceptions.UnsatisfiedException as excp: self.process_unsatisfied_exceptions(excp) parser.exit( 1, "Unable to validate the plugin requirements: {}\n".format( [x for x in excp.unsatisfied])) try: # Construct and run the plugin if constructed: renderers[args.renderer]().render(constructed.run()) except (exceptions.VolatilityException) as excp: self.process_exceptions(excp)
def main(args): dbfilename = args.file # TODO check the file exists first if not os.path.isfile(dbfilename): print("ERROR: File ", dbfilename, " does not exist.") sys.exit(-1) if args.vacuum: try: conn = sqlite3.connect(dbfilename) cur = conn.cursor() print("Vacuuming database ..") cur.execute('vacuum;') conn.commit() except sqlite3.Error as e: print("An error was encountered trying to vacuum the database:", e.args[0]) else: # TODO there are a lot more things that it might be useful to know. # These just give a tiny hint. Having NumPy would really help, # but it is not installed on Isambard2 at present. try: dburi = 'file:%s?mode=ro' % pathname2url(dbfilename) conn = sqlite3.connect(dburi, uri=True) except sqlite3.OperationalError as e: print('ERROR: Could not connect to databse: ', e) try: cur = conn.cursor() # Warning: If the database is empty, these will fail. # That should only happen if something went really wrong print('\nNumber of positions searched: ', ) cur.execute('select count(id) from position;') print(cur.fetchone()[0]) print('\nTotal number of evaluations (all visits) is: ', ) cur.execute('SELECT SUM(visits) FROM position;') print(cur.fetchone()[0]) print("\nMaximum number of visits to any single position is: ", ) cur.execute("SELECT MAX(visits) FROM position;") print(cur.fetchone()[0]) print("\nAverage number of visits per position is: ", ) cur.execute("SELECT AVG(visits) FROM position;") print(cur.fetchone()[0]) # This one doesn't produce anything useful #print("Top 10 positions by fitness (fitness,visits,id):") #cur.execute("SELECT fitness,visits,id FROM position;") #rows = cur.fetchall() #sorted_rows = sorted(rows, key=lambda fitness: fitness[0]) print("\nTop 10 most visited positions:") cur.execute("SELECT id,fitness,visits FROM position;") rows = cur.fetchall() sorted_rows = sorted(rows, key=lambda visits: visits[2]) count = 0 print('Id, Fitness, Visits:') for row in sorted_rows: count = count + 1 print(row) if count == 10: break print('\nAll best positions found:') print('Timestamp, PositionID:') for row in cur.execute('select * from global_best_history;'): print(row) # If all our fitnesses are UINT_MAX, then most likely every # attempt to assess fitness failed. That means that something # failed at all positions visited. Compiling may have failed, # execution of a wrapper script, benchmark or test may have failed, # etc. OptSearch expects quite a few failures to be encountered, # but is likely to think the search converged if we never "move" # (ie fitness does not change). # Did the fitnesses of the positions change much? cur.execute('select fitness from position;') rows = cur.fetchall() # This gives us a list of tuples print('\nFitnesses found vary between minimum: ', min(rows)[0], ' and maximum: ', max(rows)[0], '\n') except sqlite3.Error as e: print("An error was encountered trying to query the database:", e.args[0]) conn.close()
def target(): return browser.open(urljoin('file:', pathname2url(open_file)), new=self.webbrowser_open_new)
def relative_path_to_artifact_path(path): if os.path == posixpath: return path if os.path.abspath(path) == path: raise Exception("This method only works with relative paths.") return unquote(pathname2url(path))
#!/usr/bin/python3 from urllib.request import build_opener, pathname2url, urlopen, url2pathname import re import sys host = "http://ru.wikipedia.org" start_line = "mw-content-text" end_line = "references-small" bane_line = "Вики (значения)" url_line = "href=\"/wiki/" #start_url = host + pathname2url("/wiki/Конь_в_пальто_(памятник)") start_url = host + pathname2url(sys.argv[1]) pictures = [".png", ".svg", "JPG", "jpg"] picture_line = "?uselang=ru" where = {} second_start_line = "<p>" second_end_line = "</p>" steck = [] def next_site(url, num): if len(steck) - 1 >= num: steck[num] = url else: steck.append(url) if url == "http://ru.wikipedia.org/wiki/%D0%A4%D0%B8%D0%BB%D0%BE%D1%81%D0%BE%D1%84%D0%B8%D1%8F": for i in range(num + 1): print(steck[i]) print("YES, amount of steps = ", num + 1) exit(0)
def path2url(path): "Convert a pathname to a file URL" return urljoin('file:', pathname2url(path))
def url(self): """资源文件的url,供酷Q(或其他远程服务)使用""" return urljoin(hoshino.config.RES_URL, pathname2url(self.__path))
def pathname2fileurl(pathname): """Returns a file:// URL for pathname. Handles OS-specific conversions.""" return urljoin('file:', pathname2url(pathname))
def normalize_url(url, base_url=None, keep_relative=False): """ Returns a normalized URL doing a join with a base URL. URL scheme defaults to 'file' and backslashes are replaced with slashes. For file paths the os.path.join is used instead of urljoin. :param url: a relative or absolute URL. :param base_url: the reference base URL for construct the normalized URL from \ the argument. For compatibility between "os.path.join" and "urljoin" a trailing \ '/' is added to not empty paths. :param keep_relative: if set to `True` keeps relative file paths, which would \ not strictly conformant to URL format specification. :return: A normalized URL. """ def add_trailing_slash(x): return urlunsplit( (x[0], x[1], x[2] + '/' if x[2] and x[2][-1] != '/' else x[2], x[3], x[4])) def filter_url(x): x = x.strip().replace('\\', '/') while x.startswith('//'): x = x.replace('//', '/', 1) while x.startswith('file:////'): x = x.replace('file:////', 'file:///', 1) if urlsplit(x).scheme in {'', 'file'}: x = x.replace('#', '%23') return x url = filter_url(url) if base_url is not None: base_url = filter_url(base_url) base_url_parts = urlsplit(base_url) base_url = add_trailing_slash(base_url_parts) if base_url_parts.scheme not in uses_relative: base_url_parts = urlsplit('file:///{}'.format(base_url)) else: base_url_parts = urlsplit(base_url) if base_url_parts.scheme not in ('', 'file'): url = urljoin(base_url, url) else: url_parts = urlsplit(url) if url_parts.scheme not in ('', 'file'): url = urljoin(base_url, url) elif not url_parts.netloc or base_url_parts.netloc == url_parts.netloc: # Join paths only if host parts (netloc) are equal, using the os.path.join # instead of urljoin for path normalization. url = urlunsplit(( '', base_url_parts.netloc, os.path.normpath( os.path.join(base_url_parts.path, url_parts.path)), url_parts.query, url_parts.fragment, )) # Add 'file' scheme if '//' prefix is added if base_url_parts.netloc and not url.startswith(base_url_parts.netloc) \ and url.startswith('//'): url = 'file:' + url url_parts = urlsplit(url, scheme='file') if url_parts.scheme not in uses_relative: normalized_url = 'file:///{}'.format( url_parts.geturl()) # Eg. k:/Python/lib/.... elif url_parts.scheme != 'file': normalized_url = urlunsplit(( url_parts.scheme, url_parts.netloc, pathname2url(url_parts.path), url_parts.query, url_parts.fragment, )) elif os.path.isabs(url_parts.path): normalized_url = url_parts.geturl() elif keep_relative: # Can't use urlunsplit with a scheme because it converts relative paths to absolute ones. normalized_url = 'file:{}'.format(urlunsplit(('', ) + url_parts[1:])) else: normalized_url = urlunsplit(( url_parts.scheme, url_parts.netloc, os.path.abspath(url_parts.path), url_parts.query, url_parts.fragment, )) return filter_url(normalized_url)
def path_to_url(path): path = abspath(expanduser(path)) url = urljoin('file:', pathname2url(path)) log.debug("%s converted to %s", path, url) return url
def url(self): """资源文件的url,供酷Q(或其他远程服务)使用""" return urljoin(global_config.res_url, pathname2url(self.__path))
def mfa_login(request): if request.method == 'POST': form = LoginForm(request.POST) if form.is_valid(): username = form.cleaned_data['username'] password = form.cleaned_data['password'] user = authenticate(username=username.lower(), password=password) if user is not None: if user.is_active: # Get User profile up, g_o_c = UserProfile.objects.get_or_create(user=user) # If MFA, send code and redirect if up.mfa_login_mode in ("SMS", "EMAIL") and settings.MFA: # Create an MFA message mfac = MFACode.objects.create(user=up.user, mode=up.mfa_login_mode) # Send code and redirect if up.mfa_login_mode == "SMS": messages.info( request, _('An access code was sent to your mobile device. Please enter it here.' )) if up.mfa_login_mode == "EMAIL": messages.info( request, _('An access code was sent to your email. Please enter it here.' )) rev = reverse('mfa_code_confirm', args=(mfac.uid, )) # Fetch the next and urlencode if request.GET.get('next', ''): if sys.version_info[0] == 3: import urllib.request as req rev = "%s?next=%s" % (rev, req.pathname2url( request.GET.get( 'next', ''))) if sys.version_info[0] == 2: import urllib rev = "%s?next=%s" % (rev, urllib.pathname2url( request.GET.get( 'next', ''))) return HttpResponseRedirect(rev) # Else, just login as normal without MFA login(request, user) logger.info("Successful login from {}".format( get_client_ip(request))) next_param = request.GET.get('next', '') if next_param: # If a next is in the URL, then go there return HttpResponseRedirect(next_param) # otherwise just go to home. return HttpResponseRedirect(reverse('home')) else: # The user exists but is_active=False messages.error( request, _('Please check your email for a link to ' 'activate your account.')) return render(request, 'login.html', {'form': form}) else: logger.info("Invalid login attempt.") messages.error(request, _('Invalid username or password.')) return render(request, 'login.html', {'form': form}) else: return render(request, 'login.html', {'form': form}) # this is a GET return render(request, 'login.html', {'form': LoginForm()})
def sendPush(userid): user_services_url = urllib.parse.urljoin('file:', pathname2url(os.path.abspath('./wsdl_files/vipuserservices-auth-1.7.wsdl'))) user_services_client = Client(user_services_url, transport = HTTPSClientCertTransport( './privateKey_nopass.pem', './publicCert.pem')) query_services_url = urllib.parse.urljoin('file:', pathname2url(os.path.abspath('./wsdl_files/vipuserservices-query-1.7.wsdl'))) query_services_client = Client(query_services_url, transport = HTTPSClientCertTransport( './privateKey_nopass.pem', './publicCert.pem')) user_services_object = SymantecUserServices(user_services_client) query_services_object = SymantecQueryServices(query_services_client) data = {} data["displayParameters"] = [] data["displayParameters"].append({"Key":"display.message.text", "Value": message}) data["displayParameters"].append({"Key":"display.message.title", "Value": title}) response = user_services_object.authenticateUserWithPush("SupportPush",userid, None, data) status=user_services_object.getResponseValue(response,'status') statusMessage=user_services_object.getResponseValue(response,'statusMessage') if (status == '6040'): print("Push Initiated for", userid) transaction=user_services_object.getResponseValue(response,'transactionId') request=user_services_object.getResponseValue(response,'requestId') else: print("Error sending push to", userid, ":" , statusMessage) return("Error: Unable to send push to " + userid + ": " + statusMessage) isExit = False isError = False for sec in range(1,30 // 3): if isExit: break time.sleep(3) # NEED NEW SOLUTION for querying on interval in python poll_status = query_services_object.pollPushStatus(request,transaction) transaction_status = query_services_object.getResponseValue(poll_status,'transactionStatus') for tup in transaction_status: status=tup[1] msg=tup[2] if isError: print("\n\tError: " + msg) return("Error: Unable to validate user: "******"0000" in status: # ignore this first status for polling connection continue elif "7000" in status: print("SUCCESS! Push Accepted by " + userid) return("Push Accepted by "+ userid) isExit = True break elif "7001" in status: print("Waiting for response...") continue elif "7002" in status: print("Push Denied by", userid) return("Error: Push denied by "+ userid) isExit = True break else: isError = True return("Error: Request to " + userid + " timed out.")
def show(self, delete_after=20, scale=10, border=None, dark='#000', light='#fff'): # pragma: no cover """\ Displays this QR code. This method is mainly intended for debugging purposes. This method saves QR code as an image (by default with a scaling factor of 10) to a temporary file and opens it with the standard PNG viewer application or within the standard webbrowser. The temporary file is deleted afterwards (unless :paramref:`delete_after <segno.QRCode.show.delete_after>` is set to ``None``). If this method does not show any result, try to increase the :paramref:`delete_after <segno.QRCode.show.delete_after>` value or set it to ``None`` :param delete_after: Time in seconds to wait till the temporary file is deleted. :type delete_after: int or None :param int scale: Integer indicating the size of a single module. :param border: Integer indicating the size of the quiet zone. If set to ``None`` (default), the recommended border size will be used. :type border: int or None :param dark: The color of the dark modules (default: black). :param light: The color of the light modules (default: white). """ import os import time import tempfile import webbrowser import threading try: # Python 3 from urllib.parse import urljoin from urllib.request import pathname2url except ImportError: # Python 2 from urlparse import urljoin from urllib import pathname2url def delete_file(name): time.sleep(delete_after) try: os.unlink(name) except OSError: pass f = tempfile.NamedTemporaryFile('wb', suffix='.png', delete=False) try: self.save(f, scale=scale, dark=dark, light=light, border=border) except: f.close() os.unlink(f.name) raise f.close() webbrowser.open_new_tab(urljoin('file:', pathname2url(f.name))) if delete_after is not None: t = threading.Thread(target=delete_file, args=(f.name, )) t.start()
def checkIndividualSetting(setting, dirName): """Validate feature settings in a given dirName.""" if 'name' not in setting: raise ValueError('Every setting must have a name property.') if setting['name'] in previousNames: raise ValueError('Duplicate setting name: ' + setting['name']) previousNames.add(setting['name']) if 'type' not in setting: raise ValueError('Every setting must have a type property.') if setting['type'] != 'checkbox' and setting['type'] != 'select': raise ValueError( 'Only checkbox and select settings are supported at this time. Pull requests are welcome!' ) if 'title' not in setting: raise ValueError('Every setting must have a title.') if 'actions' not in setting: raise ValueError( 'Every setting must declare actions that happen when the setting is activated.' ) for action in setting['actions']: if not isinstance(setting['actions'][action], list): raise ValueError( 'Actions must be declared as an array, for example ["injectCSS", "main.css"].' ) if len(setting['actions'][action]) % 2 != 0: raise ValueError( 'Actions must have an even number of elements, for example ["injectCSS", "main.css"].' ) if setting['type'] == 'checkbox': if 'true' not in setting['actions'] and 'false' not in setting[ 'actions']: raise ValueError( 'Checkbox settings must declare an action for "true" or "false" to have any effect.' ) elif setting['type'] == 'select': if '1' not in setting['actions']: raise ValueError( 'Select settings must declare an action for "1" to have any effect.' ) # Apply the defaults. if 'section' not in setting: setting['section'] = 'general' if 'default' not in setting: setting['default'] = False if 'description' not in setting: setting['description'] = '' # Give a relative path to the files in the actions section that the settings system can understand # what URL to load when it takes an action for action in setting['actions']: i = 0 while i < len(setting['actions'][action]): currentAction = setting['actions'][action][i] if currentAction == 'injectCSS' or currentAction == 'injectScript': fullPath = os.path.join(dirName, setting['actions'][action][i + 1]) # Convert to / if we're on Windows fullPath = pathname2url(fullPath) fullPath = fullPath.replace("./src/common/", "") setting['actions'][action][i + 1] = fullPath i += 2
def open(self, path, create=True): if not has_bsddb: return NO_STORE homeDir = path if self.__identifier is None: self.__identifier = URIRef(pathname2url(abspath(homeDir))) db_env = self._init_db_environment(homeDir, create) if db_env == NO_STORE: return NO_STORE self.db_env = db_env self.__open = True dbname = None dbtype = db.DB_BTREE # auto-commit ensures that the open-call commits when transactions # are enabled dbopenflags = DBOPENFLAGS if self.transaction_aware is True: dbopenflags |= db.DB_AUTO_COMMIT if create: dbopenflags |= db.DB_CREATE dbmode = 0o660 dbsetflags = 0 # create and open the DBs self.__indicies = [ None, ] * 3 self.__indicies_info = [ None, ] * 3 for i in range(0, 3): index_name = to_key_func(i)( ("s".encode("latin-1"), "p".encode("latin-1"), "o".encode("latin-1")), "c".encode("latin-1"), ).decode() index = db.DB(db_env) index.set_flags(dbsetflags) index.open(index_name, dbname, dbtype, dbopenflags, dbmode) self.__indicies[i] = index self.__indicies_info[i] = (index, to_key_func(i), from_key_func(i)) lookup = {} for i in range(0, 8): results = [] for start in range(0, 3): score = 1 len = 0 for j in range(start, start + 3): if i & (1 << (j % 3)): score = score << 1 len += 1 else: break tie_break = 2 - start results.append(((score, tie_break), start, len)) results.sort() score, start, len = results[-1] def get_prefix_func(start, end): def get_prefix(triple, context): if context is None: yield "" else: yield context i = start while i < end: yield triple[i % 3] i += 1 yield "" return get_prefix lookup[i] = ( self.__indicies[start], get_prefix_func(start, start + len), from_key_func(start), results_from_key_func(start, self._from_string), ) self.__lookup_dict = lookup self.__contexts = db.DB(db_env) self.__contexts.set_flags(dbsetflags) self.__contexts.open("contexts", dbname, dbtype, dbopenflags, dbmode) self.__namespace = db.DB(db_env) self.__namespace.set_flags(dbsetflags) self.__namespace.open("namespace", dbname, dbtype, dbopenflags, dbmode) self.__prefix = db.DB(db_env) self.__prefix.set_flags(dbsetflags) self.__prefix.open("prefix", dbname, dbtype, dbopenflags, dbmode) self.__k2i = db.DB(db_env) self.__k2i.set_flags(dbsetflags) self.__k2i.open("k2i", dbname, db.DB_HASH, dbopenflags, dbmode) self.__i2k = db.DB(db_env) self.__i2k.set_flags(dbsetflags) self.__i2k.open("i2k", dbname, db.DB_RECNO, dbopenflags, dbmode) self.__needs_sync = False t = Thread(target=self.__sync_run) t.setDaemon(True) t.start() self.__sync_thread = t return VALID_STORE
def extract_base64_data_from_html_page(file: pathlib.Path): """ Extract all base64 data from the given HTML page and store the data in separate files. We do this by building a new HTML string using the non-base64 data pieces from the original file and the filenames we will generate for each of the base64 data strings. Base64 data is found with regex matching. Each data string is decoded and saved as a separate file. """ with file.open(encoding='utf-8') as fp: html = fp.read() # Make the subresources directory subresources_directory = file.parent / 'resources' / f'{file.stem}_resources' subresources_directory.mkdir(parents=True, exist_ok=True) offset = 0 new_html = '' filename_counter = 0 # A cache for elements that are repeated (e.g. icons) saved_strings = {} # Remove any existing `<base>` tag html = BASE_TAG_PATTERN.sub('', html) # Add `'self'` to the Content Security Policy # so we can load our extracted resources html = OLD_CSP.sub(NEW_CSP, html) base64_data_matches = BASE64_DATA_PATTERN.finditer(html) for match in base64_data_matches: # Add the content before the base64 data new_html += html[offset:match.start(1)] base64_string = match.group('string') # Check to see if we have already encountered this base64 string. # If we haven't seen it, we'll go through the process of decoding, # saving, and adding it to our cache. file_path = saved_strings.get(base64_string) if file_path is None: mime_type = match.group('mime') filename_counter += 1 filename = generate_filename(mime_type, str(filename_counter)) binary_data = base64.b64decode(base64_string) file_path = subresources_directory / filename with file_path.open('wb') as resource_file: resource_file.write(binary_data) saved_strings[base64_string] = file_path # "Replace" the old base64 data with the relative # path to the newly created file new_html += pathname2url(file_path.relative_to(file.parent).as_posix()) # Move our offset to the end of the old base64 data offset = match.end(1) # Add the remainder of the content new_html += html[offset:] return new_html
def path2url(path): # noqa: E303 return urljoin('file:', pathname2url(path))
def path2url(path): return urlparse.urljoin('file:', urllib.pathname2url(path))
def on_open_clicked(self, button): Gio.AppInfo.launch_default_for_uri( 'file:///' + pathname2url(os.path.dirname(self.selected.fullpath)), None)
def __generate_access_token(configuration, okta_response_handler): """ This function generates an access token by making a call to Okta :param ApiConfiguration configuration: The configuration to use :param typing.callable okta_response_handler: An optional function to handle the Okta response :return: RefreshingToken api_token: A refreshing API token """ # Encode credentials that may contain special characters encoded_password = pathname2url(configuration.password) encoded_client_id = pathname2url(configuration.client_id) encoded_client_secret = pathname2url(configuration.client_secret) # Prepare our authentication request token_request_body = f"grant_type=password&username={configuration.username}" \ f"&password={encoded_password}&scope=openid client groups offline_access" \ f"&client_id={encoded_client_id}&client_secret={encoded_client_secret}" headers = { "Accept": "application/json", "Content-Type": "application/x-www-form-urlencoded" } # extra request args kwargs = {"headers": headers} if configuration.proxy_config is not None: kwargs["proxies"] = configuration.proxy_config.format_proxy_schema( ) # use certificate if supplied if configuration.certificate_filename is not None: kwargs["verify"] = configuration.certificate_filename # make request to Okta to get an authentication token okta_response = requests.post(configuration.token_url, data=token_request_body, **kwargs) if okta_response_handler is not None: okta_response_handler(okta_response) # Ensure that we have a 200 response code if okta_response.status_code != 200: raise ValueError(okta_response.json()) # convert the json encoded response to be able to extract the token values okta_json = okta_response.json() # Retrieve our api token from the authentication response api_token = RefreshingToken( token_url=configuration.token_url, client_id=encoded_client_id, client_secret=encoded_client_secret, initial_access_token=okta_json["access_token"], initial_token_expiry=okta_json["expires_in"], refresh_token=okta_json["refresh_token"], proxies=kwargs.get("proxies", None), certificate_filename=kwargs.get("verify", None)) return api_token
def convert(self, remove_executed=False): """ Convert the executed notebook to a restructured text (RST) file or HTML. Parameters ---------- delete_executed : bool, optional Controls whether to remove the executed notebook or not. """ if not path.exists(self._executed_nb_path): raise IOError( "Executed notebook file doesn't exist! Expected: {0}".format( self._executed_nb_path)) if path.exists(self._output_path) and not self.overwrite: logger.debug( "{0} version of notebook already exists at {1}. Use " "overwrite=True or --overwrite (at cmd line) to re-run".format( self._output_type, self._output_path)) return self._output_path # Initialize the resources dict - see: # https://github.com/jupyter/nbconvert/blob/master/nbconvert/nbconvertapp.py#L327 resources = {} resources['config_dir'] = '' # we don't need to specify config resources['unique_key'] = self.nb_name # path to store extra files, like plots generated resources['output_files_dir'] = 'nboutput' if self.base_path is None: path_to_root = '' else: path_to_root = path.relpath(self.base_path, start=path.split(self.nb_path)[0]) path_to_root += path.sep resources['path_to_pages_root'] = request.pathname2url(path_to_root) # Exports the notebook to the output format logger.debug('Exporting notebook to {}...'.format(self._output_type)) if self._output_type == 'RST': exporter = RSTExporter() elif self._output_type == 'HTML': exporter = HTMLExporter() else: raise ValueError('This should be impossible... output_type should ' 'have been checked earlier, but it is ' 'unrecognized') if self.template_file: exporter.template_file = self.template_file output, resources = exporter.from_filename(self._executed_nb_path, resources=resources) # Write the output file writer = FilesWriter() output_file_path = writer.write(output, resources, notebook_name=self.nb_name) if self._output_type == 'RST': self._add_filter_keywords(output_file_path) if remove_executed: # optionally, clean up the executed notebook file remove(self._executed_nb_path) title = '' try: with open(self.nb_path) as f: nb = nbformat.reader.read(f) title = nb['cells'][0]['source'].split('#')[1].split( "\n")[0].strip() except Exception: print( 'Failed to parse notebook title from first cell, please check notebook.' ) page_info = dict(output_file_path=output_file_path, name=self.nb_name.replace("_", ' ').title(), title=title) return page_info
def build_cdx_request(self, request): cdx_url = self.cdx_url_template.format(url=pathname2url(request.url)) cdx_request = Request(cdx_url) cdx_request.meta['wayback_machine_original_request'] = request cdx_request.meta['wayback_machine_cdx_request'] = True return cdx_request
class Highchart(WebView): """Create a Highcharts webview widget. Parameters ---------- parent: QObject Qt parent object, if any. bridge: QObject Exposed as ``window.pybridge`` in JavaScript. options: dict Default options for this chart. See Highcharts docs. Some options are already set in the default theme. highchart: str One of `Chart`, `StockChart`, or `Map` Highcharts JS types. enable_zoom: bool Enables scroll wheel zooming and right-click zoom reset. enable_select: str If '+', allow series' points to be selected by clicking on the markers, bars or pie slices. Can also be one of 'x', 'y', or 'xy' (all of which can also end with '+' for the above), in which case it indicates the axes on which to enable rectangle selection. The list of selected points for each input series (i.e. a list of arrays) is passed to the ``selection_callback``. Each selected point is represented as its index in the series. If the selection is empty, the callback parameter is a single empty list. javascript: str Additional JavaScript code to evaluate beforehand. If you need something exposed in the global namespace, assign it as an attribute to the ``window`` object. debug: bool Enables right-click context menu and inspector tools. **kwargs: The additional options. The underscores in argument names imply hierarchy, e.g., keyword argument such as ``chart_type='area'`` results in the following object, in JavaScript:: { chart: { type: 'area' } } The original `options` argument is updated with options from these kwargs-derived objects. """ _HIGHCHARTS_HTML = urljoin( 'file:', pathname2url(join(join(dirname(__file__), '_highcharts'), 'chart.html'))) def __init__(self, parent=None, bridge=None, options=None, *, highchart='Chart', enable_zoom=False, enable_select=False, selection_callback=None, javascript='', debug=False, **kwargs): options = (options or {}).copy() enable_select = enable_select or '' if not isinstance(options, dict): raise ValueError('options must be dict') if enable_select not in ('', '+', 'x', 'y', 'xy', 'x+', 'y+', 'xy+'): raise ValueError("enable_select must be '+', 'x', 'y', or 'xy'") if enable_select and not selection_callback: raise ValueError('enable_select requires selection_callback') super().__init__(parent, bridge, debug=debug, url=QUrl(self._HIGHCHARTS_HTML)) self.debug = debug self.highchart = highchart self.enable_zoom = enable_zoom enable_point_select = '+' in enable_select enable_rect_select = enable_select.replace('+', '') if enable_zoom: _merge_dicts( options, _kwargs_options( dict(mapNavigation_enableMouseWheelZoom=True, mapNavigation_enableButtons=False))) if enable_select: self._selection_callback = selection_callback self.frame.addToJavaScriptWindowObject('__highchart', self) _merge_dicts( options, _kwargs_options( dict(chart_events_click='/**/unselectAllPoints/**/'))) if enable_point_select: _merge_dicts( options, _kwargs_options( dict(plotOptions_series_allowPointSelect=True, plotOptions_series_point_events_click= '/**/clickedPointSelect/**/'))) if enable_rect_select: _merge_dicts( options, _kwargs_options( dict(chart_zoomType=enable_rect_select, chart_events_selection='/**/rectSelectPoints/**/'))) if kwargs: _merge_dicts(options, _kwargs_options(kwargs)) super_evalJS = super().evalJS def evalOptions(): super_evalJS(javascript) self.evalJS(''' var options = {options}; fixupOptionsObject(options); Highcharts.setOptions(options); '''.format(options=json(options))) self.frame.loadFinished.connect(evalOptions) # Give above scripts time to load qApp.processEvents(QEventLoop.ExcludeUserInputEvents) qApp.processEvents(QEventLoop.ExcludeUserInputEvents) def contextMenuEvent(self, event): """ Zoom out on right click. Also disable context menu.""" if self.enable_zoom: self.evalJS('chart.zoomOut();') if self.debug: super().contextMenuEvent(event) @staticmethod def _JSObject_factory(obj): pyqt_type = type(obj).__mro__[-2] if isinstance(obj, (list, np.ndarray)): pyqt_type = 'QVariantList' elif isinstance(obj, Mapping): pyqt_type = 'QVariantMap' else: raise TypeError("Can't expose object of type {}. Too easy. Use " "evalJS method instead.".format(type(obj))) class _JSObject(QObject): """ This class hopefully prevent options data from being marshalled into a string-like dumb (JSON) object when passed into JavaScript. """ def __init__(self, parent, obj): super().__init__(parent) self._obj = obj @pyqtProperty(pyqt_type) def _options(self): return self._obj return _JSObject def exposeObject(self, name, obj): """Expose the object `obj` as ``window.<name>`` in JavaScript. If the object contains any string values that start and end with literal ``/**/``, those are evaluated as JS expressions the result value replaces the string in the object. The exposure, as defined here, represents a snapshot of object at the time of execution. Any future changes on the original Python object are not (necessarily) visible in its JavaScript counterpart. Parameters ---------- name: str The global name the object is exposed as. obj: object The object to expose. Must contain only primitive types, such as: int, float, str, bool, list, dict, set, numpy.ndarray. """ try: obj = _to_primitive_types(obj) except TypeError: raise TypeError('object must consist of primitive types ' '(allowed: int, float, str, bool, list, ' 'dict, set, numpy.ndarray, ...)') from None pydata = self._JSObject_factory(obj)(self, obj) self.frame.addToJavaScriptWindowObject('_' + name, pydata) self.evalJS(''' window.{0} = window._{0}._options; fixupOptionsObject({0}); '''.format(name)) def chart(self, options=None, *, highchart=None, javascript='', javascript_after='', **kwargs): """ Populate the webview with a new Highcharts JS chart. Parameters ---------- options, highchart, javascript, **kwargs: The parameters are the same as for the object constructor. javascript_after: str Same as `javascript`, except that the code is evaluated after the chart, available as ``window.chart``, is created. Notes ----- Passing ``{ series: [{ data: some_data }] }``, if ``some_data`` is a numpy array, it is **more efficient** to leave it as numpy array instead of converting it ``some_data.tolist()``, which is done implicitly. """ # Give default options some time to apply qApp.processEvents(QEventLoop.ExcludeUserInputEvents) options = (options or {}).copy() if not isinstance(options, MutableMapping): raise ValueError('options must be dict') if kwargs: _merge_dicts(options, _kwargs_options(kwargs)) self.exposeObject('pydata', options) highchart = highchart or self.highchart or 'Chart' self.evalJS(''' {javascript}; window.chart = new Highcharts.{highchart}(pydata); {javascript_after}; '''.format( javascript=javascript, javascript_after=javascript_after, highchart=highchart, )) def evalJS(self, javascript): """ Asynchronously evaluate JavaScript code. """ # Why do we need this async? I don't know. But performance of # loading/evaluating any JS code is greatly improved this way. _ASYNC = 'setTimeout(function() { %s; }, 10);' super().evalJS(_ASYNC % javascript) def clear(self): """Remove all series from the chart""" self.evalJS(''' if (window.chart) { while(chart.series.length > 0) { chart.series[0].remove(false); } chart.redraw(); } ''') @pyqtSlot('QVariantList') def _on_selected_points(self, points): self._selection_callback( [np.sort(selected).astype(int) for selected in points])
def rel_url(self): rel_path = os.path.relpath(self._path(), os.path.dirname(settings.MEDIA_ROOT)) return pathname2url(rel_path)
def as_uri(self, scheme='ssh'): return '{0}://{1}{2}'.format(scheme, self.remote._fqhost, urllib.pathname2url(str(self)))
def path_to_url(path): if six.PY2: path = path.encode('utf-8') return pathname2url(path)
def as_uri(self, scheme='file'): return urlparse.urljoin( str(scheme) + ':', urllib.pathname2url(str(self)))
def get_adf15( self, element: str, charge: str, filetype: str, year="", ) -> DataArray: """Read data from the specified ADF15 ADAS file. Implementation is capable of reading files with compact and expanded formatting e.g. pec96][ne_pju][ne9.dat and pec40][ar_cl][ar16.dat respectively Parameters ---------- element The atomic symbol for the element which will be retrieved. charge Charge state of the ion (e.g. 16 for Ar 16+), can also include other string for more complicated path (transport_llu][ar15ic.dat setting charge to "15ic") filetype The type of data to retrieve. Options: ic, cl, ca, ls, llu, ... year The two-digit year label for the data. = "transport" if special transport path Returns ------- : The data in the specified file. Dimensions are density and temperature. Each members of the dataset correspond to a different charge state. """ def explicit_reshape(data_to_reshape, nd, nt): data = np.empty((nd, nt)) for id in range(nd): for it in range(nt): data[id, it] = data_to_reshape[id * nt + it] return data def build_file_component(year, element): file_component = "transport" if year != "transport": file_component = f"pec{year}][{element.lower()}" return file_component def file_type(identifier): identifier_dict = { "+": "compact", ":": "expanded", } file_type = identifier_dict.get(identifier) if file_type is None: raise ValueError( f"Unknown file header identified ({identifier}).") return file_type def transition_match(transition_line): transition_type = "orbitals" match = ( r"c\s+(\d+.)" # isel r"\s+(\d+.\d+)" # wavelength r"\s+(\d+)(\(\d\)\d\(.+\d?.\d\))-" # transition upper level r".+(\d+)(\(\d\)\d\(.+\d?.\d\))" # transition lower level ) header_re = re.compile(match) m = header_re.search(transition_line) if not m: transition_type = "n_levels" match = r"c\s+(\d+.)\s+(\d+.\d+)\s+([n]\=.\d+.-.[n]\=.\d+)" header_re = re.compile(match) m = header_re.search(transition_line) if not m: raise ValueError( f"Unknown transition formatting ({identifier}).") return transition_type, match now = datetime.datetime.now() file_component = build_file_component(year, element) filename = Path(pathname2url(file_component)) / pathname2url( f"{file_component}_{filetype.lower()}]" f"[{element.lower()}{charge.lower()}.dat") header_match = { "compact": r"(\d+).+/(\S+).*\+(.*)photon", "expanded": r"(\d+).+/(\S+).*\:(.*)photon", } section_header_match = { "compact": r"(\d+.\d+).+\s+(\d+)\s+(\d+).+type\s?" r"=\s?(\S+).+isel.+\s+(\d+)", "expanded": r"(\d+.\d+)\s+(\d+)\s+(\d+).+type\s?=" r"\s?(\S+).+isel\s+?=\s+?(\d+)", } with self._get_file("adf15", filename) as f: header = f.readline().strip().lower() identifier = file_type(header.split("/")[1][2]) match = header_match[identifier] m = re.search(match, header, re.I) assert isinstance(m, re.Match) ntrans = int(m.group(1)) element_name = m.group(2).strip().lower() charge_state = int(m.group(3)) assert element_name == element.lower() m = re.search(r"(\d+)(\S*)", charge) assert isinstance(m, re.Match) extracted_charge = m.group(1) if charge_state != int(extracted_charge): raise ValueError( f"Charge state in ADF15 file ({charge_state}) does not " f"match argument ({charge}).") # Read first section header to build arrays outside of reading loop match = section_header_match[identifier] header_re = re.compile(match) m = None while not m: line = f.readline().strip().lower() m = header_re.search(line) assert isinstance(m, re.Match) nd = int(m.group(2)) nt = int(m.group(3)) ttype: List[str] = [] tindex = np.empty(ntrans) wavelength = np.empty(ntrans) # Read Photon Emissivity Coefficient rates data = np.empty((ntrans, nd, nt)) for i in range(ntrans): m = header_re.search(line) assert isinstance(m, re.Match) assert int(m.group(5)) - 1 == i tindex[i] = i + 1 ttype.append(m.group(4)) wavelength[i] = float(m.group(1)) # (Angstroms) densities = np.fromfile(f, float, nd, " ") temperatures = np.fromfile(f, float, nt, " ") data_tmp = np.fromfile(f, float, nd * nt, " ") data[i, :, :] = explicit_reshape(data_tmp, nd, nt) line = f.readline().strip().lower() data = np.transpose(np.array(data), (0, 2, 1)) # Read Transition information from end of file file_end_re = re.compile(r"c\s+[isel].+\s+[transition].+\s+[type]") while not file_end_re.search(line): line = f.readline().strip().lower() _ = f.readline() if identifier == "expanded": _ = f.readline() line = f.readline().strip().lower() transition_type, match = transition_match(line) transition_re = re.compile(match) format_transition = { "orbitals": lambda m: f"{m.group(4)}-{m.group(6)}".replace(" ", ""), "n_levels": lambda m: m.group(3).replace(" ", ""), } transition = [] for i in tindex: m = transition_re.search(line) assert isinstance(m, re.Match) assert int(m.group(1)[:-1]) == i transition_tmp = format_transition[transition_type](m) transition.append(transition_tmp) line = f.readline().strip().lower() gen_type = ADF15_GENERAL_DATATYPES[filetype] spec_type = element name = f"{spec_type}_{gen_type}" attrs = { "datatype": (gen_type, spec_type), "provenance": self.create_provenance(filename, now), } coords = [ ("index", tindex), ("electron_temperature", temperatures), # eV ("electron_density", densities * 10**6), # m**-3 ] pecs = DataArray( data * 10**-6, coords=coords, name=name, attrs=attrs, ) # Add extra dimensions attached to index pecs = pecs.assign_coords(wavelength=("index", wavelength)) # (A) pecs = pecs.assign_coords( transition=("index", transition) ) # (2S+1)L(w-1/2)-(2S+1)L(w-1/2) of upper-lower levels, no blank spaces pecs = pecs.assign_coords(type=("index", ttype)) # (excit, recomb, cx) return pecs