def download_file(url, local_file): try: start_time = datetime.now() dir_name = os.path.dirname(local_file) url_path = urlpath.URL(url) if (os.path.isdir(dir_name) is 0): os.makedirs(dir_name) headers = requests.utils.default_headers() headers.update({ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36' }) r = requests.get(url, headers=headers, timeout=3) file_pure_path = urlpath.URL(local_file) local_file = file_pure_path.drive + file_pure_path.path with open(local_file, "wb") as code: code.write(r.content) end_time = datetime.now() print('下载完成:{0} ==> {1},耗时:{2}s'.format( url, local_file, (end_time - start_time).seconds)) except Exception as e: print('下载出错...' + str(e))
def loading(store=None): if store is None: raise ValueError('data store not specified') if store == 'gs': base = urlpath.URL('gs://') elif store == 'az': base = urlpath.URL('https://carbonplan.blob.core.windows.net') elif store == 'local': base = pathlib.Path(pathlib.Path.home() / 'workdir') return base
def download( url, folder_name ): #for downloading the file, takes url of the image and folder to be created dir = r"C:\Users\Sid\Downloads" + "/FilmyCart/" + folder_name os.makedirs(dir, exist_ok=True) #create dir if not exist name = dir + "/" + str(urlpath.URL(url).name) if not os.path.exists(name): #check if file already exist print("Downloading " + str(urlpath.URL(url).name)) urllib.request.urlretrieve(url, name) #download the image return 1 #successful else: return 0 #unsuccessful
def open_file(path, mode=None): uri = urlpath.URL(path) if not uri.scheme or uri.scheme == 'file://': path = pathlib.Path(uri.path) else: raise NotImplementedError("Cannot open non-local file") return file_ext_handlers.get(path.suffix, open)( str(path), mode) if mode else file_ext_handlers[path.suffix](str(path))
def get_user_snapshot_result(user_id, snapshot, result): maybe_uri = _get_snapshot(user_id, snapshot).get(result, None) if maybe_uri is None: abort(404) thing = _get_uri(maybe_uri) if not thing: return maybe_uri return str(urlpath.URL(request.base_url) / 'data')
def combine_url(refurl, path): path_low = path.lower() if (path_low.startswith('http://') or path_low.startswith('https://')): return path url = urlpath.URL(refurl) if (path_low.startswith('/')): return "{0}{1}".format(url.drive, path) else: return "{0}/{1}".format(url.parent, path)
def test_load_save_measure(self): """ Coverage: * KnowledgeGraph() constructor * KnowledgeGraph.load_rdf() from pathlib.Path, urlpath.URL * KnowledgeGraph.safe_rdf() * KnowledgeGraph.load_jsonld() * KnowledgeGraph.save_jsonld() * Measure() constructor * Measure.measure_graph() * Measure.get_node_count() """ tmp = tempfile.NamedTemporaryFile(mode="w+b", delete=False) try: with warnings.catch_warnings(): warnings.filterwarnings("ignore", message="unclosed file*") # load RDF from urlpath.URL kg = kglab.KnowledgeGraph() path = urlpath.URL( "https://storage.googleapis.com/kglab-tutorial/foaf.rdf") kg.load_rdf(path, format="xml") tmp.close() # save RDF to local file reference kg.save_rdf(tmp.name) tmp.close() # load RDF from pathlib.Path kg = kglab.KnowledgeGraph() path = pathlib.Path(tmp.name) kg.load_rdf(path) tmp.close() # save JSON-LD to local file reference kg.save_jsonld(tmp.name) tmp.close() # load JSON-LD from pathlib.Path kg = kglab.KnowledgeGraph() path = pathlib.Path(tmp.name) kg.load_jsonld(path) # measure graph measure = kglab.Measure() measure.measure_graph(kg) # verify self.assertTrue(measure.get_node_count() == 35) self.assertTrue(measure.get_edge_count() == 62) finally: os.unlink(tmp.name) tmp.close()
def get_consumer(url, handlers=None, auto_start=False): url = urlpath.URL(url) if not url.scheme == SCHEME: return None handlers = handlers or {} cons = RabbitQueueConsumer( pika.ConnectionParameters(host=url.hostname, port=url.port), handlers) if auto_start: cons.start() return cons
def get_driver(uri, driver=None): """ Find and initialize driver for an asset :param uri: (string) URI to the asset :param driver: (optional, string) name of the required driver """ suffix = urlpath.URL(uri).suffix driver = driver or suffix2driver.get(suffix) if driver is None: raise ValueError(f'Unknown driver for suffix: {suffix}') driver = drivers.get(driver) if driver is None: raise ValueError(f'Unknown driver: {driver}') return driver(uri)
def get_dispatcher(url, topics, auto_start=True, **kwargs): """ creates a dispatcher and connects to the given url :param auto_start: bool: start the dispatcher right away (on False, this will not connect the dispatcher) :param url: where to connect :param topics: topics that could be published :param kwargs: anything to pass on to the dispatcher implementation (TODO) :return: a dispatcher if the url scheme matches, None otherwise """ url = urlpath.URL(url) if not url.scheme == SCHEME: return None topics = topics if not isinstance(topics, str) else (topics, ) dispatcher = RabbitQueueDispatcher( pika.ConnectionParameters(host=url.hostname, port=url.port), topics) if auto_start: dispatcher.start() return dispatcher
def start(cls, host, port): url = urlpath.URL().with_scheme(cls.SCHEME).with_hostinfo(host, port) out = cls(url) out.get_config() return out
def open(cls, path, mode='rb'): path = pathlib.Path(urlpath.URL(path).path) return cls(path.open(mode), path)
def uri(self): return str(urlpath.URL().with_scheme(f"file://").with_netloc(str(self.path)))
def open(uri: URI, config: Optional[tiledb.Config] = None) -> Segy: uri = urlpath.URL(uri) if not isinstance(uri, PurePath) else uri ts = open2(uri / "data", uri / "headers", config) ts._uri = uri return ts
def test_get_config_sets_configuration(sessionserver, config_dict): ses = ClientHTTPSession(urlpath.URL(sessionserver.url_for("/"))) ses.get_config() assert ses._server_config == config_dict
def base_64_url_for(data): return urlpath.URL().with_scheme("base64").with_hostinfo( base64.encodebytes(data).encode('utf-8'))
def get_server(api_host, api_port, server_name="server_gui", static_dir=None): static_dir = static_dir or Path(__file__).parent / 'static' template_dir = Path(static_dir).parent / 'templates' app = Flask(server_name, static_folder=str(static_dir), template_folder=str(template_dir)) api_url = urlpath.URL().with_scheme("http").with_hostinfo( api_host, api_port) root_cards_view = dict(SETUP_CARDS_FUNCTION="setup_user_cards()", CARDS_DIV_NAME='user_cards_div', BRAND_LOGO_CLASS="") user_cards_view = dict(SETUP_CARDS_FUNCTION="setup_user_option_cards()", CARDS_DIV_NAME='user_options_div', BRAND_LOGO_CLASS="fa fa-angle-left") snapshot_card_view = dict(SETUP_CARDS_FUNCTION="setup_snapshot_cards()", CARDS_DIV_NAME='user_snapshots_div', BRAND_LOGO_CLASS="fa fa-angle-left") user_location_view = dict(BRAND_LOGO_CLASS="fa fa-angle-left", CHART_NAME='3dgraph') user_feelings_view = dict(BRAND_LOGO_CLASS="fa fa-angle-left", CHART_NAME='linegraph') user_snapshot_view = dict() # these are the options a user gets when they go into a user menu # TODO: aggregate automatically user_option_list = ['locations', 'feelings', 'snapshots'] @app.route('/js/cortex.js') def setup_cortex(): return render_template('cortex.js', API_URL=str(api_url), user_option_list=user_option_list) @app.route("/") def index(): return render_template('cards_view.html', **root_cards_view) @app.route("/user") def user(): return render_template('cards_view.html', **user_cards_view) @app.route("/user/snapshots") def user_snapshots(): return render_template('cards_view.html', **snapshot_card_view) @app.route("/user/locations") def user_locations(): return render_template('location_view.html', **user_location_view) @app.route("/user/feelings") def user_feelings(): return render_template('feelings_view.html', **user_feelings_view) @app.route("/user/snapshot") def user_snapshot(): return render_template('snapshot_view.html', **user_snapshot_view) @app.after_request def set_caching(r): r.headers["Cache-Control"] = "no-cache, no-store, must-revalidate" r.headers["Pragma"] = "no-cache" r.headers["Expires"] = "0" r.headers['Cache-Control'] = 'public, max-age=0' return r return app
def __init__(self, host, port): self.url = urlpath.URL().with_scheme('http').with_hostinfo(host, port)
def _get_uri(thing, schemes=EXPOSED_SCHEMES): if not isinstance(thing, (bytes, str, urlpath.URL)): return None candidate = urlpath.URL(thing) return candidate if candidate.scheme in schemes else None
def __init__(self, url, server_config=None): self._server_config = server_config self.url = urlpath.URL(url)
def get_database(url): url = urlpath.URL(url) if url.scheme != SCHEME: return None return MongoDB.connect(url.with_scheme(''))
def connect(cls, url): url = urlpath.URL(url) return cls(pymongo.MongoClient(host=url.hostname, port=url.port), url)
def copy_asset(catalog, asset_key, update_catalog=False, item_id=None, to_uri=None, filesystem_from=None, filesystem_to=None, max_workers=None): """ Download an asset for (one of) the items of a catalog :param catalog: (:class:`~pystac.Catalog`) input catalog :param asset_key: (str) asset key :param update_catalog: (bool) update the catalog links to the new asset location (default: False) :param item_id: (optional, str) item ID (default: retrieve assets for all items of the catalog) :param to_uri: (optional, str) URI of the folder where to save the assets (default: the catalog's item directories) :param filesystem_from: (optional, `fsspec` compatible FileSystem instance) file system for input source :param filesystem_to: (optional, `fsspec` compatible FileSystem instance) file system for output destination :param max_workers: (optional, int) number of processes that will be used to copy the assets (default to number of processors) """ root_href = catalog.get_self_href() if root_href is None and to_uri is None: raise ValueError('Provide URI where to save the assets ' '(or save the catalog to disk)') if item_id is not None: item = catalog.get_item(item_id, recursive=True) if item is not None: items = (item,) else: raise ValueError(f'Item not found: {item_id}') else: items = catalog.get_all_items() with ProcessPoolExecutor(max_workers=max_workers, mp_context=mp_context) \ as executor: future_to_asset = {} for item in items: asset = item.assets.get(asset_key) if asset is None: raise ValueError(f'Asset {asset_key} not found for {item.id}') if to_uri is not None: destination = urlpath.URL(to_uri) / item.id else: destination = urlpath.URL(item.get_self_href()).parent future = executor.submit( copy, source=asset.get_absolute_href(), dest=destination, filesystem_from=filesystem_from, filesystem_to=filesystem_to, ) future_to_asset[future] = asset for future in as_completed(future_to_asset): new_href = future.result() if update_catalog: asset = future_to_asset[future] asset.href = new_href # update link in catalog
import pathlib import urlpath __all__ = ['path_list'] base = urlpath.URL( 'https://www.lmsal.com/solarsoft/irisa/data/level2_compressed') path_list = [ pathlib.Path( '2013/09/30/20130930Mosaic/IRISMosaic_20130930_Si1393.fits.gz'), pathlib.Path( '2013/10/13/20131013Mosaic/IRISMosaic_20131013_Si1393.fits.gz'), pathlib.Path( '2013/10/21/20131021Mosaic/IRISMosaic_20131021_Si1393.fits.gz'), pathlib.Path( '2013/10/27/20131027Mosaic/IRISMosaic_20131027_Si1393.fits.gz'), pathlib.Path( '2014/03/17/20140317Mosaic/IRISMosaic_20140317_Si1393.fits.gz'), pathlib.Path( '2014/03/24/20140324Mosaic/IRISMosaic_20140324_Si1393.fits.gz'), pathlib.Path( '2014/05/12/20140512Mosaic/IRISMosaic_20140512_Si1393.fits.gz'), pathlib.Path( '2014/05/27/20140527Mosaic/IRISMosaic_20140527_Si1393.fits.gz'), pathlib.Path( '2014/06/23/20140623Mosaic/IRISMosaic_20140623_Si1393.fits.gz'), pathlib.Path( '2014/07/27/20140727Mosaic/IRISMosaic_20140727_Si1393.fits.gz'), pathlib.Path( '2014/08/24/20140824Mosaic/IRISMosaic_20140824_Si1393.fits.gz'),