def s3_inventory_data_file(): return ( URL(INVENTORY_FOLDER) / URL(INVENTORY_BUCKET_NAME) / URL("data") / INVENTORY_DATA_FILE )
def s3_inventory_manifest_file(): return ( URL(INVENTORY_FOLDER) / URL(INVENTORY_BUCKET_NAME) / "2021-09-17T00-00Z" / INVENTORY_MANIFEST_FILE )
def gen_url_option(str_opt, set_site=set_site, set_runcontrol=set_runcontrol, set_initcond=set_initcond, source='docs'): '''construct a URL for option based on source :param str_opt: option name, defaults to '' :param str_opt: str, optional :param source: URL source: 'docs' for readthedocs.org; 'github' for github repo, defaults to 'docs' :param source: str, optional :return: a valid URL pointing to the option related resources :rtype: urlpath.URL ''' dict_base = { 'docs': URL('https://suews-docs.readthedocs.io/en/latest/input_files/'), 'github': URL('https://github.com/Urban-Meteorology-Reading/SUEWS-Docs/raw/master/docs/source/input_files/' ), } url_base = dict_base[source] url_page = choose_page(str_opt, set_site, set_runcontrol, set_initcond, source=source) # print('str_opt', str_opt, url_base, url_page) str_opt_x = form_option(str_opt) url_opt = url_base / (url_page + str_opt_x) return url_opt
def test_with(self): url = URL('http://www.example.com/path/to/file.exe?query?fragment') self.assertEqual( str(url.with_scheme('https')), 'https://www.example.com/path/to/file.exe?query?fragment') self.assertEqual(str(url.with_netloc('localhost')), 'http://localhost/path/to/file.exe?query?fragment') self.assertEqual( str(url.with_userinfo('username', 'password')), 'http://*****:*****@www.example.com/path/to/file.exe?query?fragment' ) self.assertEqual( str(url.with_userinfo(None, None)), 'http://www.example.com/path/to/file.exe?query?fragment') self.assertEqual( str(url.with_hostinfo('localhost', 8080)), 'http://localhost:8080/path/to/file.exe?query?fragment') self.assertEqual(str(URL('http://example.com/base/') / 'path/to/file'), 'http://example.com/base/path/to/file') self.assertEqual( str( URL('http://example.com/path/?q') / URL('http://localhost/app/?q') / URL('to/content')), 'http://localhost/app/to/content')
def named_contexts(self) -> Dict[str, LDContext]: """Reusable named contexts.""" return { 'github-generated': as_document(URL('file://', self.plugin_data_dir / 'gh-auto.yaml'), ), 'github': as_document(URL('file://', self.plugin_data_dir / 'gh.yaml'), ), }
def test_idempotent(self): url = URL( 'http://\u65e5\u672c\u8a9e\u306e.\u30c9\u30e1\u30a4\u30f3.jp/' 'path/to/\u30d5\u30a1\u30a4\u30eb.ext?\u30af\u30a8\u30ea') self.assertEqual(url, URL(str(url))) self.assertEqual( url, URL('http://xn--u9ju32nb2abz6g.xn--eckwd4c7c.jp/' 'path/to/\u30d5\u30a1\u30a4\u30eb.ext?\u30af\u30a8\u30ea'))
def test_trailing_sep(self): original = 'http://www.example.com/path/with/trailing/sep/' url = URL(original) self.assertEqual(str(url), original) self.assertEqual(url.name, 'sep') self.assertEqual(url.parts[-1], 'sep') self.assertEqual(URL('htp://example.com/').trailing_sep, '') self.assertEqual(URL('htp://example.com/with/sep/').trailing_sep, '/') self.assertEqual(URL('htp://example.com/without/sep').trailing_sep, '') self.assertEqual( URL('htp://example.com/with/double-sep//').trailing_sep, '//')
def _get_details(path, data): """ Extract details from the metadata returned by the dCache API :param path: (str) file or directory path :param data: (dict) metadata as provided by the API :return (dict) parsed metadata """ path = URL(path) name = data.get('fileName') # fileName might be missing name = path/name if name is not None else path name = name.path element_type = data.get('fileType') element_type = DCACHE_FILE_TYPES.get(element_type, 'other') created = data.get('creationTime') # in ms created = datetime.fromtimestamp(created / 1000.) modified = data.get('mtime') # in ms modified = datetime.fromtimestamp(modified / 1000.) return dict( name=name, size=data.get('size'), type=element_type, created=created, modified=modified )
def __init__( self, fs, url, mode="rb", asynchronous=False, session=None, loop=None, **kwargs ): path = fs._strip_protocol(url) url = URL(fs.webdav_url) / path self.url = url.as_uri() self.details = {"name": self.url, "size": None} self.asynchronous = asynchronous self.session = session self.loop = loop super(HTTPStreamFile, self).__init__( fs=fs, path=path, mode=mode, block_size=0, cache_type="none", cache_options={}, **kwargs) if self.mode == "rb": self.r = sync(self.loop, self.session.get, self.url, **self.kwargs) elif self.mode == "wb": pass else: raise ValueError
def named_contexts(self) -> Dict[str, LDContext]: """Reusable named contexts.""" return { 'prov': as_document( URL('file://', self.plugin_data_dir / 'named-context.yaml'), ), }
def _record(self, op: _Ops) -> Tuple[bool, int]: mypage_url = URL('https://attendance.moneyforward.com/my_page') # 1. GET my_page and scrape authenticity_token cookies = {'_session_id': self._sess.session_id} mypage = req.request('GET', mypage_url, cookies=cookies) if not mypage.ok: return False, mypage.status_code form_input = bs(mypage.content.decode()).find('input', attrs={'value': op.value}) if form_input is None: raise RuntimeError(f'"{_Ops.to_human_readable(op)}" is not available') token = form_input.parent.find('input', attrs={'name': 'authenticity_token'}).attrs['value'] # 2. POST time record web_time_recorder_url = path / 'my_page' / 'web_time_recorder' cookies = {'_session_id': self._sess.session_id} d = datetime.utcnow() form = { 'authenticity_token': token, 'web_time_recorder_form[event]': op.value, 'web_time_recorder_form[date]': f'{d.year}/{d.month}/{d.day}', 'web_time_recorder_form[user_time]': d.strftime('%Y-%m-%dT%H:%M:%S.000Z'), 'web_time_recorder_form[office_location_id]': self._sess.location_id, } recorder = req.request('POST', web_time_recorder_url, cookies=cookies, data=form) return recorder.ok, recorder.status_code
def __init__( self, fs, url, mode="rb", block_size=None, cache_type="bytes", cache_options=None, asynchronous=False, session=None, loop=None, **kwargs ): path = fs._strip_protocol(url) url = URL(fs.webdav_url) / path self.url = url.as_uri() self.asynchronous = asynchronous self.session = session self.loop = loop if mode not in {"rb", "wb"}: raise ValueError super(HTTPFile, self).__init__( fs=fs, path=path, mode=mode, block_size=block_size, cache_type=cache_type, cache_options=cache_options, **kwargs )
class ProdConfig(Config): ''' Build configuration: freezing to static files and serving locally ''' ENV = 'production' IMG_DOMAIN = URL("http://localhost:5003/") # Manually add fonts to list to incorporate into freezer FONTS = { # tuple: vf, latin, woff2 'Roboto_Slab': 'RobotoSlab-VariableFont_wght-Latin.woff2', 'Public_Sans': 'PublicSans-VariableFont_wght-Min.woff2', } # Frozen Flask config FREEZER_DESTINATION_IGNORE = ['404.html', 'netlify.toml'] FREEZER_STATIC_IGNORE = [ 'fonts/', 'scss/', 'img/', 'css/', 'favicon/', 'js/', '.DS_Store', ] FREEZER_IGNORE_404_NOT_FOUND = True CSS_OUT_DIR = Path('website', 'build', 'static', 'css') # Flask-HTMLmin MINIFY_HTML = True
def _make_auth(self): config_path = self.config_file_path.parent.expanduser() if not config_path.exists(): self.log.info("Creating directory %s", config_path) config_path.mkdir(parents=True) config_file = self.config_file_path if not config_file.exists(): self.log.info("Creating config file %s", config_path) url = input("Enter your website url: ") key = input("Enter your OAuth Client Key: ") secret = input("Enter OAuth Client Secret: ") user_info = {"url": url, "key": key, "secret": secret} config_file.write_text(yaml.dump(user_info)) else: user_info = yaml.safe_load(config_file.read_text()) for param in ["url", "key", "secret"]: if param not in user_info: raise AssertionError( f"Not found '{param}' in {config_file}") base_url = URL(user_info["url"]).resolve() api_endpoint = str(base_url / "wp-json/wp/v2") request_token_url = str(base_url / "oauth1/request") authorization_base_url = str(base_url / "oauth1/authorize") access_token_url = str(base_url / "oauth1/access") if "owner_key" not in user_info: oauth = OAuth1Session(user_info["key"], client_secret=user_info["secret"]) fetch_response = oauth.fetch_request_token(request_token_url) resource_owner_key = fetch_response.get("oauth_token") resource_owner_secret = fetch_response.get("oauth_token_secret") authorization_url = oauth.authorization_url(authorization_base_url) print(authorization_url) verifier = input("Visit authorization_url. And input verifier.") # Using OAuth1Session oauth = OAuth1Session( user_info["key"], client_secret=user_info["secret"], resource_owner_key=resource_owner_key, resource_owner_secret=resource_owner_secret, verifier=verifier, ) oauth_tokens = oauth.fetch_access_token(access_token_url) resource_owner_key = oauth_tokens.get("oauth_token") resource_owner_secret = oauth_tokens.get("oauth_token_secret") user_info["owner_key"] = resource_owner_key user_info["owner_secret"] = resource_owner_secret config_file.write_text(yaml.dump(user_info)) else: resource_owner_key = user_info["owner_key"] resource_owner_secret = user_info["owner_secret"] self.api_endpoint = base_url / "wp-json/wp/v2" self.oauth = OAuth1Session( user_info["key"], client_secret=user_info["secret"], resource_owner_key=resource_owner_key, resource_owner_secret=resource_owner_secret, )
def create_and_upload_stac(cog_file: Path, s3_dst: str, year) -> Item: out_path = URL(f"{s3_dst}/{year}/") log.info("Item base creation") item = create_stac_item( str(cog_file), id=str(odc_uuid("gmw", "2.0", [cog_file.name.replace("tif", "")])), with_proj=True, input_datetime=datetime(int(year), 12, 31), properties={ "odc:product": "gmw", "start_datetime": f"{year}-01-01T00:00:00Z", "end_datetime": f"{year}-12-31T23:59:59Z", }, ) log.info("links creation") item.set_self_href(str(out_path / f"gmw_{year}_stac-item.json")) item.add_links([ pystac.Link( target=str(SOURCE_URL_PATH / FILE_NAME.format(year=year)), title="Source file", rel=pystac.RelType.DERIVED_FROM, media_type="application/zip", ) ]) out_data = out_path / cog_file.name # Remove asset created by create_stac_item and add our own del item.assets["asset"] item.assets["mangrove"] = pystac.Asset( href=str(out_data), title="gmw-v1.0", media_type=pystac.MediaType.COG, roles=["data"], ) log.info(f"Item created {item.to_dict()}") log.info(f"Item validated {item.validate()}") log.info(f"Dump the data to S3 {str(cog_file)}") s3_dump( data=open(str(cog_file), "rb").read(), url=str(out_data), ACL="bucket-owner-full-control", ContentType="image/tiff", ) log.info(f"File written to {out_data}") log.info("Write STAC to S3") s3_dump( data=json.dumps(item.to_dict(), indent=2), url=item.self_href, ACL="bucket-owner-full-control", ContentType="application/json", ) log.info(f"STAC written to {item.self_href}") return item
def test_query_field_order(self): url = URL('http://example.com/').with_query(field1='field1', field2='field2', field3='field3') self.assertEqual( str(url), 'http://example.com/?field1=field1&field2=field2&field3=field3')
def contexts_by_url(self, url: URL) -> Iterable[LDContext]: """Yield all contexts by URL.""" ancestor_directories = self.ancestors_by_url(url) for directory in ancestor_directories: for filename in self.context_filenames: if (context_file := directory / filename).exists(): yield self.as_jsonld_document( url=URL(f'file://{context_file}'), )
def upload_image_link(wp, boost_note_root, image_link): actual_src = image_link["src"].replace( ":storage", str(boost_note_root / "attachments")) result = wp.upload_image(actual_src, name=image_link["text"]) result = result.json() url = URL(result["guid"]["rendered"]) url.path return url.path
def readable_status(self) -> str: """Render status in a readable form.""" url = URL(str(self.status)) if url.scheme == 'local': return url.path return str(url)
def __init__(self, config: dict): _conf = copy.deepcopy(config) # build url for this connection corename = _conf.pop("corename") _conf["url"] = str(URL(_conf["url"]) / corename) self.con = pysolr.Solr(**_conf)
def get_spec_from_api(api_base_url): """Get the spec and version at the given source.""" spec_url = URL(api_base_url) / "swagger/v2/swagger.json" spec_response = spec_url.get() spec = spec_response.json() version = fetch_orbit_spec_version_no(api_base_url) return spec, version
def url_parts(self) -> Tuple[str, str]: """Parse the GitHub URL.""" try: _hostname, owner_name, repo_name, *etc = URL(self.url).parts except ValueError as err: raise ValueError( f'{self.url} is not a valid GitHub repo URL.', ) from err return owner_name, repo_name
def __call__(self, url: str, options: PyLDOptions) -> PyLDResponse: """Compile document for PyLD.""" return { 'document': self.as_jsonld_document( url=URL(url), iri=url, ), 'contextUrl': None, }
def fetch_orbit_spec_version_no(api_base_url): """Get the Orbit API spec version number from the given source.""" spec_version_url = URL(api_base_url) / "About/Version" spec_version_response = spec_version_url.get() spec_version = spec_version_response.json().get("version") if spec_version is None: raise LookupError("Failed to fetch OrbitAPI spec version number.") else: return spec_version
def test_good_inputs(self, path): """ GIVEN a valid domain, path and list of widths WHEN write_srcset is called THEN check output is the expected srcset string """ srcset = sv.write_srcset(URL('/domain/'), path, [0, 1, 10, 100, 1000, 10_000]) assert srcset == "/domain/test_0.jpg 0w, /domain/test_1.jpg 1w, /domain/test_10.jpg 10w, /domain/test_100.jpg 100w, /domain/test_1000.jpg 1000w, /domain/test_10000.jpg 10000w"
class DevConfig(Config): ''' Development configuration: running local flask server ''' # from run.py ENV = 'development' DEBUG = True TEMPLATES_AUTO_RELOAD = True IMG_DOMAIN = URL("/static/img/out/") VIEW_TEST = True
def test_join(self): url = URL('http://www.example.com/path/to/file.ext?query#fragment') self.assertEqual(str(url / 'https://secure.example.com/path'), 'https://secure.example.com/path') self.assertEqual(str(url / '/changed/path'), 'http://www.example.com/changed/path') self.assertEqual(str(url.with_name('other_file')), 'http://www.example.com/path/to/other_file')
def _get_webdav_url(cls, path): """ Extract kwargs encoded in the path(s) :param path: (str or list) if list, extract URL from the first element :return (dict) """ if isinstance(path, list): return cls._get_webdav_url(path[0]) return URL(path).drive or None
def test_auth_window_on_url_changed_is_redirect(auth_window, url): m = mock.MagicMock() auth_window.on_redirect_url.connect(m) qurl = QtCore.QUrl(url) auth_window.on_url_changed(qurl) m.assert_called_once_with(URL(url)) auth_window._browser.close.assert_called_once()
def _strip_protocol(cls, path): """ Turn path from fully-qualified to file-system-specific :param path: (str or list) :return (str) """ if isinstance(path, list): return [cls._strip_protocol(p) for p in path] return URL(path).path