def download_file(url, local_file):
try:
start_time = datetime.now()
dir_name = os.path.dirname(local_file)
url_path = urlpath.URL(url)
if (os.path.isdir(dir_name) is 0):
os.makedirs(dir_name)
headers = requests.utils.default_headers()
headers.update({
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
})
r = requests.get(url, headers=headers, timeout=3)
file_pure_path = urlpath.URL(local_file)
local_file = file_pure_path.drive + file_pure_path.path
with open(local_file, "wb") as code:
code.write(r.content)
end_time = datetime.now()
print('下载完成:{0} ==> {1},耗时:{2}s'.format(
url, local_file, (end_time - start_time).seconds))
except Exception as e:
print('下载出错...' + str(e))
def loading(store=None):
if store is None:
raise ValueError('data store not specified')
if store == 'gs':
base = urlpath.URL('gs://')
elif store == 'az':
base = urlpath.URL('https://carbonplan.blob.core.windows.net')
elif store == 'local':
base = pathlib.Path(pathlib.Path.home() / 'workdir')
return base
def download(
url, folder_name
): #for downloading the file, takes url of the image and folder to be created
dir = r"C:\Users\Sid\Downloads" + "/FilmyCart/" + folder_name
os.makedirs(dir, exist_ok=True) #create dir if not exist
name = dir + "/" + str(urlpath.URL(url).name)
if not os.path.exists(name): #check if file already exist
print("Downloading " + str(urlpath.URL(url).name))
urllib.request.urlretrieve(url, name) #download the image
return 1 #successful
else:
return 0 #unsuccessful
def open_file(path, mode=None):
uri = urlpath.URL(path)
if not uri.scheme or uri.scheme == 'file://':
path = pathlib.Path(uri.path)
else:
raise NotImplementedError("Cannot open non-local file")
return file_ext_handlers.get(path.suffix, open)(
str(path), mode) if mode else file_ext_handlers[path.suffix](str(path))
def get_user_snapshot_result(user_id, snapshot, result):
maybe_uri = _get_snapshot(user_id, snapshot).get(result, None)
if maybe_uri is None:
abort(404)
thing = _get_uri(maybe_uri)
if not thing:
return maybe_uri
return str(urlpath.URL(request.base_url) / 'data')
def combine_url(refurl, path):
path_low = path.lower()
if (path_low.startswith('http://') or path_low.startswith('https://')):
return path
url = urlpath.URL(refurl)
if (path_low.startswith('/')):
return "{0}{1}".format(url.drive, path)
else:
return "{0}/{1}".format(url.parent, path)
def test_load_save_measure(self):
"""
Coverage:
* KnowledgeGraph() constructor
* KnowledgeGraph.load_rdf() from pathlib.Path, urlpath.URL
* KnowledgeGraph.safe_rdf()
* KnowledgeGraph.load_jsonld()
* KnowledgeGraph.save_jsonld()
* Measure() constructor
* Measure.measure_graph()
* Measure.get_node_count()
"""
tmp = tempfile.NamedTemporaryFile(mode="w+b", delete=False)
try:
with warnings.catch_warnings():
warnings.filterwarnings("ignore", message="unclosed file*")
# load RDF from urlpath.URL
kg = kglab.KnowledgeGraph()
path = urlpath.URL(
"https://storage.googleapis.com/kglab-tutorial/foaf.rdf")
kg.load_rdf(path, format="xml")
tmp.close()
# save RDF to local file reference
kg.save_rdf(tmp.name)
tmp.close()
# load RDF from pathlib.Path
kg = kglab.KnowledgeGraph()
path = pathlib.Path(tmp.name)
kg.load_rdf(path)
tmp.close()
# save JSON-LD to local file reference
kg.save_jsonld(tmp.name)
tmp.close()
# load JSON-LD from pathlib.Path
kg = kglab.KnowledgeGraph()
path = pathlib.Path(tmp.name)
kg.load_jsonld(path)
# measure graph
measure = kglab.Measure()
measure.measure_graph(kg)
# verify
self.assertTrue(measure.get_node_count() == 35)
self.assertTrue(measure.get_edge_count() == 62)
finally:
os.unlink(tmp.name)
tmp.close()
def get_consumer(url, handlers=None, auto_start=False):
url = urlpath.URL(url)
if not url.scheme == SCHEME:
return None
handlers = handlers or {}
cons = RabbitQueueConsumer(
pika.ConnectionParameters(host=url.hostname, port=url.port), handlers)
if auto_start:
cons.start()
return cons
def get_driver(uri, driver=None):
"""
Find and initialize driver for an asset
:param uri: (string) URI to the asset
:param driver: (optional, string) name of the required driver
"""
suffix = urlpath.URL(uri).suffix
driver = driver or suffix2driver.get(suffix)
if driver is None:
raise ValueError(f'Unknown driver for suffix: {suffix}')
driver = drivers.get(driver)
if driver is None:
raise ValueError(f'Unknown driver: {driver}')
return driver(uri)
def get_dispatcher(url, topics, auto_start=True, **kwargs):
"""
creates a dispatcher and connects to the given url
:param auto_start: bool: start the dispatcher right away (on False, this will not connect the dispatcher)
:param url: where to connect
:param topics: topics that could be published
:param kwargs: anything to pass on to the dispatcher implementation (TODO)
:return: a dispatcher if the url scheme matches, None otherwise
"""
url = urlpath.URL(url)
if not url.scheme == SCHEME:
return None
topics = topics if not isinstance(topics, str) else (topics, )
dispatcher = RabbitQueueDispatcher(
pika.ConnectionParameters(host=url.hostname, port=url.port), topics)
if auto_start:
dispatcher.start()
return dispatcher
def start(cls, host, port):
url = urlpath.URL().with_scheme(cls.SCHEME).with_hostinfo(host, port)
out = cls(url)
out.get_config()
return out
def open(cls, path, mode='rb'):
path = pathlib.Path(urlpath.URL(path).path)
return cls(path.open(mode), path)
def uri(self):
return str(urlpath.URL().with_scheme(f"file://").with_netloc(str(self.path)))
def open(uri: URI, config: Optional[tiledb.Config] = None) -> Segy:
uri = urlpath.URL(uri) if not isinstance(uri, PurePath) else uri
ts = open2(uri / "data", uri / "headers", config)
ts._uri = uri
return ts
def test_get_config_sets_configuration(sessionserver, config_dict):
ses = ClientHTTPSession(urlpath.URL(sessionserver.url_for("/")))
ses.get_config()
assert ses._server_config == config_dict
def base_64_url_for(data):
return urlpath.URL().with_scheme("base64").with_hostinfo(
base64.encodebytes(data).encode('utf-8'))
def get_server(api_host, api_port, server_name="server_gui", static_dir=None):
static_dir = static_dir or Path(__file__).parent / 'static'
template_dir = Path(static_dir).parent / 'templates'
app = Flask(server_name,
static_folder=str(static_dir),
template_folder=str(template_dir))
api_url = urlpath.URL().with_scheme("http").with_hostinfo(
api_host, api_port)
root_cards_view = dict(SETUP_CARDS_FUNCTION="setup_user_cards()",
CARDS_DIV_NAME='user_cards_div',
BRAND_LOGO_CLASS="")
user_cards_view = dict(SETUP_CARDS_FUNCTION="setup_user_option_cards()",
CARDS_DIV_NAME='user_options_div',
BRAND_LOGO_CLASS="fa fa-angle-left")
snapshot_card_view = dict(SETUP_CARDS_FUNCTION="setup_snapshot_cards()",
CARDS_DIV_NAME='user_snapshots_div',
BRAND_LOGO_CLASS="fa fa-angle-left")
user_location_view = dict(BRAND_LOGO_CLASS="fa fa-angle-left",
CHART_NAME='3dgraph')
user_feelings_view = dict(BRAND_LOGO_CLASS="fa fa-angle-left",
CHART_NAME='linegraph')
user_snapshot_view = dict()
# these are the options a user gets when they go into a user menu
# TODO: aggregate automatically
user_option_list = ['locations', 'feelings', 'snapshots']
@app.route('/js/cortex.js')
def setup_cortex():
return render_template('cortex.js',
API_URL=str(api_url),
user_option_list=user_option_list)
@app.route("/")
def index():
return render_template('cards_view.html', **root_cards_view)
@app.route("/user")
def user():
return render_template('cards_view.html', **user_cards_view)
@app.route("/user/snapshots")
def user_snapshots():
return render_template('cards_view.html', **snapshot_card_view)
@app.route("/user/locations")
def user_locations():
return render_template('location_view.html', **user_location_view)
@app.route("/user/feelings")
def user_feelings():
return render_template('feelings_view.html', **user_feelings_view)
@app.route("/user/snapshot")
def user_snapshot():
return render_template('snapshot_view.html', **user_snapshot_view)
@app.after_request
def set_caching(r):
r.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
r.headers["Pragma"] = "no-cache"
r.headers["Expires"] = "0"
r.headers['Cache-Control'] = 'public, max-age=0'
return r
return app
def __init__(self, host, port):
self.url = urlpath.URL().with_scheme('http').with_hostinfo(host, port)
def _get_uri(thing, schemes=EXPOSED_SCHEMES):
if not isinstance(thing, (bytes, str, urlpath.URL)):
return None
candidate = urlpath.URL(thing)
return candidate if candidate.scheme in schemes else None
def __init__(self, url, server_config=None):
self._server_config = server_config
self.url = urlpath.URL(url)
def get_database(url):
url = urlpath.URL(url)
if url.scheme != SCHEME:
return None
return MongoDB.connect(url.with_scheme(''))
def connect(cls, url):
url = urlpath.URL(url)
return cls(pymongo.MongoClient(host=url.hostname, port=url.port), url)
def copy_asset(catalog, asset_key, update_catalog=False, item_id=None,
to_uri=None, filesystem_from=None, filesystem_to=None,
max_workers=None):
"""
Download an asset for (one of) the items of a catalog
:param catalog: (:class:`~pystac.Catalog`) input catalog
:param asset_key: (str) asset key
:param update_catalog: (bool) update the catalog links to the new asset
location (default: False)
:param item_id: (optional, str) item ID (default: retrieve assets for all
items of the catalog)
:param to_uri: (optional, str) URI of the folder where to save the assets
(default: the catalog's item directories)
:param filesystem_from: (optional, `fsspec` compatible FileSystem instance)
file system for input source
:param filesystem_to: (optional, `fsspec` compatible FileSystem instance)
file system for output destination
:param max_workers: (optional, int) number of processes that will be used
to copy the assets (default to number of processors)
"""
root_href = catalog.get_self_href()
if root_href is None and to_uri is None:
raise ValueError('Provide URI where to save the assets '
'(or save the catalog to disk)')
if item_id is not None:
item = catalog.get_item(item_id, recursive=True)
if item is not None:
items = (item,)
else:
raise ValueError(f'Item not found: {item_id}')
else:
items = catalog.get_all_items()
with ProcessPoolExecutor(max_workers=max_workers, mp_context=mp_context) \
as executor:
future_to_asset = {}
for item in items:
asset = item.assets.get(asset_key)
if asset is None:
raise ValueError(f'Asset {asset_key} not found for {item.id}')
if to_uri is not None:
destination = urlpath.URL(to_uri) / item.id
else:
destination = urlpath.URL(item.get_self_href()).parent
future = executor.submit(
copy,
source=asset.get_absolute_href(),
dest=destination,
filesystem_from=filesystem_from,
filesystem_to=filesystem_to,
)
future_to_asset[future] = asset
for future in as_completed(future_to_asset):
new_href = future.result()
if update_catalog:
asset = future_to_asset[future]
asset.href = new_href # update link in catalog
import pathlib
import urlpath
__all__ = ['path_list']
base = urlpath.URL(
'https://www.lmsal.com/solarsoft/irisa/data/level2_compressed')
path_list = [
pathlib.Path(
'2013/09/30/20130930Mosaic/IRISMosaic_20130930_Si1393.fits.gz'),
pathlib.Path(
'2013/10/13/20131013Mosaic/IRISMosaic_20131013_Si1393.fits.gz'),
pathlib.Path(
'2013/10/21/20131021Mosaic/IRISMosaic_20131021_Si1393.fits.gz'),
pathlib.Path(
'2013/10/27/20131027Mosaic/IRISMosaic_20131027_Si1393.fits.gz'),
pathlib.Path(
'2014/03/17/20140317Mosaic/IRISMosaic_20140317_Si1393.fits.gz'),
pathlib.Path(
'2014/03/24/20140324Mosaic/IRISMosaic_20140324_Si1393.fits.gz'),
pathlib.Path(
'2014/05/12/20140512Mosaic/IRISMosaic_20140512_Si1393.fits.gz'),
pathlib.Path(
'2014/05/27/20140527Mosaic/IRISMosaic_20140527_Si1393.fits.gz'),
pathlib.Path(
'2014/06/23/20140623Mosaic/IRISMosaic_20140623_Si1393.fits.gz'),
pathlib.Path(
'2014/07/27/20140727Mosaic/IRISMosaic_20140727_Si1393.fits.gz'),
pathlib.Path(
'2014/08/24/20140824Mosaic/IRISMosaic_20140824_Si1393.fits.gz'),