def run(self):
SERVER = os.environ["PY4SCI_SERVER"]
USER = os.environ["PY4SCI_USER"]
import getpass
from ftplib import FTP
from astropy.utils.console import ProgressBar
ftp = FTP(SERVER)
ftp.login(user=USER, passwd=getpass.getpass())
ftp.cwd('/public_html/PY4SCI_WS_2013_14')
for slides in ProgressBar.iterate(
glob.glob('lectures/*.html') + ['lectures/custom.css'] +
glob.glob('problems/data/*') + glob.glob('problems/*.html') +
glob.glob('practice/data/*') + glob.glob('practice/*.html')):
try:
remote_size = ftp.size(slides)
except:
remote_size = None
local_size = os.path.getsize(slides)
if local_size != remote_size:
ftp.storbinary('STOR ' + slides, open(slides, 'rb'))
ftp.storbinary('STOR index.html', open('index.html', 'rb'))
ftp.quit()
def run(self):
SERVER = os.environ["PY4SCI_SERVER"]
USER = os.environ["PY4SCI_USER"]
import getpass
from ftplib import FTP
from astropy.utils.console import ProgressBar
ftp = FTP(SERVER)
ftp.login(user=USER, passwd=getpass.getpass())
ftp.cwd('/public_html/PY4SCI_WS_2013_14')
for slides in ProgressBar.iterate(glob.glob('lectures/data/*')
+ glob.glob('lectures/*.html')
+ ['lectures/custom.css']
+ glob.glob('problems/data/*')
+ glob.glob('problems/*.html')
+ glob.glob('practice/data/*')
+ glob.glob('practice/*.html')):
try:
remote_size = ftp.size(slides)
except:
remote_size = None
local_size = os.path.getsize(slides)
if local_size != remote_size:
ftp.storbinary('STOR ' + slides, open(slides, 'rb'))
ftp.storbinary('STOR index.html', open('index.html', 'rb'))
ftp.quit()
def run(self):
SERVER = os.environ["PY4SCI_SERVER"]
USER = os.environ["PY4SCI_USER"]
import getpass
from ftplib import FTP
from astropy.utils.console import ProgressBar
ftp = FTP(SERVER)
ftp.login(user=USER, passwd=getpass.getpass())
ftp.cwd('/public_html/astropy4herts')
for slides in ProgressBar.iterate(glob.glob('notebooks/data/*')
+ glob.glob('notebooks/*.html')):
try:
remote_size = ftp.size(slides)
except:
remote_size = None
local_size = os.path.getsize(slides)
if local_size != remote_size:
ftp.storbinary('STOR ' + slides, open(slides, 'rb'))
ftp.storbinary('STOR notebooks.html', open('notebooks.html', 'rb'))
ftp.quit()
def make_validation_report(
urls=None, destdir='astropy.io.votable.validator.results',
multiprocess=True, stilts=None):
"""
Validates a large collection of web-accessible VOTable files.
Generates a report as a directory tree of HTML files.
Parameters
----------
urls : list of strings, optional
If provided, is a list of HTTP urls to download VOTable files
from. If not provided, a built-in set of ~22,000 urls
compiled by HEASARC will be used.
destdir : path, optional
The directory to write the report to. By default, this is a
directory called ``'results'`` in the current directory. If the
directory does not exist, it will be created.
multiprocess : bool, optional
If `True` (default), perform validations in parallel using all
of the cores on this machine.
stilts : path, optional
To perform validation with ``votlint`` from the the Java-based
`STILTS <http://www.star.bris.ac.uk/~mbt/stilts/>`_ VOTable
parser, in addition to `astropy.io.votable`, set this to the
path of the ``'stilts.jar'`` file. ``java`` on the system shell
path will be used to run it.
Notes
-----
Downloads of each given URL will be performed only once and cached
locally in *destdir*. To refresh the cache, remove *destdir*
first.
"""
from astropy.utils.console import (color_print, ProgressBar, Spinner)
if stilts is not None:
if not os.path.exists(stilts):
raise ValueError(
'{0} does not exist.'.format(stilts))
destdir = os.path.abspath(destdir)
if urls is None:
with Spinner('Loading URLs', 'green') as s:
urls = get_urls(destdir, s)
else:
color_print('Marking URLs', 'green')
for url in ProgressBar.iterate(urls):
with result.Result(url, root=destdir) as r:
r['expected'] = type
args = [(url, destdir) for url in urls]
color_print('Downloading VO files', 'green')
ProgressBar.map(
download, args, multiprocess=multiprocess)
color_print('Validating VO files', 'green')
ProgressBar.map(
validate_vo, args, multiprocess=multiprocess)
if stilts is not None:
color_print('Validating with votlint', 'green')
votlint_args = [(stilts, x, destdir) for x in urls]
ProgressBar.map(
votlint_validate, votlint_args, multiprocess=multiprocess)
color_print('Generating HTML files', 'green')
ProgressBar.map(
write_html_result, args, multiprocess=multiprocess)
with Spinner('Grouping results', 'green') as s:
subsets = result.get_result_subsets(urls, destdir, s)
color_print('Generating index', 'green')
html.write_index(subsets, urls, destdir)
color_print('Generating subindices', 'green')
subindex_args = [(subset, destdir, len(urls)) for subset in subsets]
ProgressBar.map(
write_subindex, subindex_args, multiprocess=multiprocess)
def make_validation_report(urls=None,
destdir='astropy.io.votable.validator.results',
multiprocess=True,
stilts=None):
"""
Validates a large collection of web-accessible VOTable files.
Generates a report as a directory tree of HTML files.
Parameters
----------
urls : list of strings, optional
If provided, is a list of HTTP urls to download VOTable files
from. If not provided, a built-in set of ~22,000 urls
compiled by HEASARC will be used.
destdir : path, optional
The directory to write the report to. By default, this is a
directory called ``'results'`` in the current directory. If the
directory does not exist, it will be created.
multiprocess : bool, optional
If `True` (default), perform validations in parallel using all
of the cores on this machine.
stilts : path, optional
To perform validation with ``votlint`` from the the Java-based
`STILTS <http://www.star.bris.ac.uk/~mbt/stilts/>`_ VOTable
parser, in addition to `astropy.io.votable`, set this to the
path of the ``'stilts.jar'`` file. ``java`` on the system shell
path will be used to run it.
Notes
-----
Downloads of each given URL will be performed only once and cached
locally in *destdir*. To refresh the cache, remove *destdir*
first.
"""
from astropy.utils.console import (color_print, ProgressBar, Spinner)
if stilts is not None:
if not os.path.exists(stilts):
raise ValueError('{0} does not exist.'.format(stilts))
destdir = os.path.abspath(destdir)
if urls is None:
with Spinner('Loading URLs', 'green') as s:
urls = get_urls(destdir, s)
else:
color_print('Marking URLs', 'green')
for url in ProgressBar.iterate(urls):
with result.Result(url, root=destdir) as r:
r['expected'] = type
args = [(url, destdir) for url in urls]
color_print('Downloading VO files', 'green')
ProgressBar.map(download, args, multiprocess=multiprocess)
color_print('Validating VO files', 'green')
ProgressBar.map(validate_vo, args, multiprocess=multiprocess)
if stilts is not None:
color_print('Validating with votlint', 'green')
votlint_args = [(stilts, x, destdir) for x in urls]
ProgressBar.map(votlint_validate,
votlint_args,
multiprocess=multiprocess)
color_print('Generating HTML files', 'green')
ProgressBar.map(write_html_result, args, multiprocess=multiprocess)
with Spinner('Grouping results', 'green') as s:
subsets = result.get_result_subsets(urls, destdir, s)
color_print('Generating index', 'green')
html.write_index(subsets, urls, destdir)
color_print('Generating subindices', 'green')
subindex_args = [(subset, destdir, len(urls)) for subset in subsets]
ProgressBar.map(write_subindex, subindex_args, multiprocess=multiprocess)
