def install_custom_css( destdir, cssfile, resource=PKGNAME ):
"""
Add the kernel CSS to custom.css
"""
ensure_dir_exists( destdir )
custom = os.path.join( destdir, 'custom.css' )
prefix = css_frame_prefix(resource)
# Check if custom.css already includes it. If so, let's remove it first
exists = False
if os.path.exists( custom ):
with io.open(custom) as f:
for line in f:
if line.find( prefix ) >= 0:
exists = True
break
if exists:
remove_custom_css( destdir, resource )
# Fetch the CSS file
cssfile += '.css'
data = pkgutil.get_data( resource, os.path.join('resources',cssfile) )
# get_data() delivers encoded data, str (Python2) or bytes (Python3)
# Add the CSS at the beginning of custom.css
# io.open uses unicode strings (unicode in Python2, str in Python3)
with io.open(custom + '-new', 'wt', encoding='utf-8') as fout:
fout.write( u'{}START ======================== */\n'.format(prefix))
fout.write( data.decode('utf-8') )
fout.write( u'{}END ======================== */\n'.format(prefix))
if os.path.exists( custom ):
with io.open( custom, 'rt', encoding='utf-8' ) as fin:
for line in fin:
fout.write( unicode(line) )
os.rename( custom+'-new',custom)
def download_images(self, color, num_images):
if num_images > 19:
raise ValueError('Number of images to download '
'must be less than 20.')
query = color + ' color'
url = 'http://www.bing.com/images/search?q=' + query
soup = self.get_soup(url)
images = [
a['src']
for a in soup.find_all('img', {'src': re.compile('mm.bing.net')})
]
images = images[:num_images]
download_directory = 'images/' + color
ensure_dir_exists(download_directory)
for img in images:
raw_img = urllib2.urlopen(img).read()
image = np.asarray(bytearray(raw_img), dtype="uint8")
image = cv2.imdecode(image, cv2.IMREAD_COLOR)
x_size, y_size, _ = image.shape
cropped_image = image[0:y_size,
int(x_size * .18):int(x_size * 1.18)]
cntr = len(
[i for i in os.listdir(download_directory) if color in i]) + 1
filename = ("images/" + color + '/' + color + "_" + str(cntr) +
'.jpg')
cv2.imwrite(filename, cropped_image)
def _ipython_dir_changed(self, name, old, new):
if old is not None:
str_old = py3compat.cast_bytes_py2(os.path.abspath(old),
sys.getfilesystemencoding()
)
if str_old in sys.path:
sys.path.remove(str_old)
str_path = py3compat.cast_bytes_py2(os.path.abspath(new),
sys.getfilesystemencoding()
)
sys.path.append(str_path)
ensure_dir_exists(new)
readme = os.path.join(new, 'README')
readme_src = os.path.join(
get_ipython_package_dir(), u'config', u'profile', 'README')
if not os.path.exists(readme) and os.path.exists(readme_src):
shutil.copy(readme_src, readme)
for d in ('extensions', 'nbextensions'):
path = os.path.join(new, d)
try:
ensure_dir_exists(path)
except OSError:
# this will not be EEXIST
self.log.error("couldn't create path %s: %s", path, e)
self.log.debug("IPYTHONDIR set to: %s" % new)
def get_logger(
name: str,
log_dir_path='.',
logging_level=logging.DEBUG,
log_format="%(asctime)s - %(process)d - %(name)s - %(levelname)s - %(message)s",
append=False,
file_ext='log'):
"""
Convenient method to get a file-based logger.
:param name: the name of the logger; also used as the main name for the log file.
:param log_dir_path: the path to the directory containing the log file; if the log file does not exist, a new file will be created; if the log file already exists, then it is overwritten if `append` is `False`, or new log lines are appended to the existing file if `append` is set `True`.
:param logging_level: provides the logging level; the default is the lowest level `logging.DEBUG`.
:param log_format: the format for each logging message; by default it includes logging time, process id, logger name, logging level and the message; check https://docs.python.org/3/library/logging.html#logrecord-attributes for more about logging format directives.
:param append: `True` if appending new log lines to the log file; `False` if the existing log file should be overwritten.
:param file_ext: the extension name for the log file.
:return: a file-based logger.
"""
logger = logging.getLogger(name)
ensure_dir_exists(log_dir_path)
handler = logging.FileHandler(
path.join(log_dir_path, f'{name}.{file_ext if file_ext else "log"}'),
'a+' if append else 'w+')
handler.setFormatter(logging.Formatter(log_format))
logger.addHandler(handler)
logger.setLevel(logging_level)
return logger
def install_custom_css(destdir, cssfile, resource=PKGNAME):
"""
Add the kernel CSS to custom.css
"""
ensure_dir_exists(destdir)
custom = os.path.join(destdir, 'custom.css')
prefix = css_frame_prefix(resource)
# Check if custom.css already includes it. If so, let's remove it first
exists = False
if os.path.exists(custom):
with io.open(custom) as f:
for line in f:
if line.find(prefix) >= 0:
exists = True
break
if exists:
remove_custom_css(destdir, resource)
# Fetch the CSS file
cssfile += '.css'
data = pkgutil.get_data(resource, os.path.join('resources', cssfile))
# get_data() delivers encoded data, str (Python2) or bytes (Python3)
# Add the CSS at the beginning of custom.css
# io.open uses unicode strings (unicode in Python2, str in Python3)
with io.open(custom + '-new', 'wt', encoding='utf-8') as fout:
fout.write(u'{}START ======================== */\n'.format(prefix))
fout.write(data.decode('utf-8'))
fout.write(u'{}END ======================== */\n'.format(prefix))
if os.path.exists(custom):
with io.open(custom, 'rt', encoding='utf-8') as fin:
for line in fin:
fout.write(unicode(line))
os.rename(custom + '-new', custom)
def subdivide_CBT(source, destination, x=4, y=4):
"""Utility function for subdividing the RGB images from the
Coloured Brodatz Textures dataset.
Parameters
----------
source : string
Full path of the folder where the original images are stored.
destination : string
Full path of the folder where the subimages will be saved.
x : int
Number of vertical subdivisions.
y : int
Number of horizontal subdivisions.
"""
from skimage import io
from IPython.utils.path import ensure_dir_exists
ensure_dir_exists(destination)
for n in range(1, 113):
imgpath = os.path.join(source, f'D{n}_COLORED.tif')
img = io.imread(imgpath)
folder = os.path.join(destination, f'D{n:03}')
ensure_dir_exists(folder)
rows = np.int_(np.linspace(0, img.shape[0], x + 1))
cols = np.int_(np.linspace(0, img.shape[1], y + 1))
for i, (start_row, end_row) in enumerate(zip(rows[:-1], rows[1:])):
for j, (start_col, end_col) in enumerate(zip(cols[:-1], cols[1:])):
sub = img[start_row:end_row, start_col:end_col, :]
sample = 1 + i*y + j
subname = f'D{n}_COLORED_{sample}.tif'
subpath = os.path.join(folder, subname)
io.imsave(subpath, sub)
def install_extension(self, url, filename=None):
"""Download and install an IPython extension.
If filename is given, the file will be so named (inside the extension
directory). Otherwise, the name from the URL will be used. The file must
have a .py or .zip extension; otherwise, a ValueError will be raised.
Returns the full path to the installed file.
"""
# Ensure the extension directory exists
ensure_dir_exists(self.ipython_extension_dir)
if os.path.isfile(url):
src_filename = os.path.basename(url)
copy = copyfile
else:
# Deferred imports
try:
from urllib.parse import urlparse # Py3
from urllib.request import urlretrieve
except ImportError:
from urlparse import urlparse
from urllib import urlretrieve
src_filename = urlparse(url).path.split('/')[-1]
copy = urlretrieve
if filename is None:
filename = src_filename
if os.path.splitext(filename)[1] not in ('.py', '.zip'):
raise ValueError("The file must have a .py or .zip extension", filename)
filename = os.path.join(self.ipython_extension_dir, filename)
copy(url, filename)
return filename
def install_extension(self, url, filename=None):
"""Download and install an IPython extension.
If filename is given, the file will be so named (inside the extension
directory). Otherwise, the name from the URL will be used. The file must
have a .py or .zip extension; otherwise, a ValueError will be raised.
Returns the full path to the installed file.
"""
# Ensure the extension directory exists
ensure_dir_exists(self.ipython_extension_dir)
if os.path.isfile(url):
src_filename = os.path.basename(url)
copy = copyfile
else:
# Deferred imports
try:
from urllib.parse import urlparse # Py3
from urllib.request import urlretrieve
except ImportError:
from urlparse import urlparse
from urllib import urlretrieve
src_filename = urlparse(url).path.split('/')[-1]
copy = urlretrieve
if filename is None:
filename = src_filename
if os.path.splitext(filename)[1] not in ('.py', '.zip'):
raise ValueError("The file must have a .py or .zip extension", filename)
filename = os.path.join(self.ipython_extension_dir, filename)
copy(url, filename)
return filename
def _ipython_dir_changed(self, change):
old = change['old']
new = change['new']
if old is not Undefined:
str_old = os.path.abspath(old)
if str_old in sys.path:
sys.path.remove(str_old)
if self.add_ipython_dir_to_sys_path:
str_path = os.path.abspath(new)
sys.path.append(str_path)
ensure_dir_exists(new)
readme = os.path.join(new, "README")
readme_src = os.path.join(
get_ipython_package_dir(), "config", "profile", "README"
)
if not os.path.exists(readme) and os.path.exists(readme_src):
shutil.copy(readme_src, readme)
for d in ("extensions", "nbextensions"):
path = os.path.join(new, d)
try:
ensure_dir_exists(path)
except OSError as e:
# this will not be EEXIST
self.log.error("couldn't create path %s: %s", path, e)
self.log.debug("IPYTHONDIR set to: %s" % new)
def _ipython_dir_changed(self, change):
old = change['old']
new = change['new']
if old is not Undefined:
str_old = py3compat.cast_bytes_py2(os.path.abspath(old),
sys.getfilesystemencoding())
if str_old in sys.path:
sys.path.remove(str_old)
str_path = py3compat.cast_bytes_py2(os.path.abspath(new),
sys.getfilesystemencoding())
sys.path.append(str_path)
ensure_dir_exists(new)
readme = os.path.join(new, 'README')
readme_src = os.path.join(get_ipython_package_dir(), u'config',
u'profile', 'README')
if not os.path.exists(readme) and os.path.exists(readme_src):
shutil.copy(readme_src, readme)
for d in ('extensions', 'nbextensions'):
path = os.path.join(new, d)
try:
ensure_dir_exists(path)
except OSError as e:
# this will not be EEXIST
self.log.error("couldn't create path %s: %s", path, e)
self.log.debug("IPYTHONDIR set to: %s" % new)
def init_assignment(self, assignment_id, student_id):
super(AutogradeApp, self).init_assignment(assignment_id, student_id)
# try to get the student from the database, and throw an error if it
# doesn't exist
gb = Gradebook(self.db_url)
try:
gb.find_student(student_id)
except MissingEntry:
if self.create_student:
self.log.warning("Creating student with ID '%s'", student_id)
gb.add_student(student_id)
else:
self.fail("No student with ID '%s' exists in the database", student_id)
# try to read in a timestamp from file
src_path = self._format_source(assignment_id, student_id)
timestamp = self._get_existing_timestamp(src_path)
if timestamp:
submission = gb.update_or_create_submission(
assignment_id, student_id, timestamp=timestamp)
self.log.info("%s submitted at %s", submission, timestamp)
# if the submission is late, print out how many seconds late it is
if timestamp and submission.total_seconds_late > 0:
self.log.warning("%s is %s seconds late", submission, submission.total_seconds_late)
else:
submission = gb.update_or_create_submission(assignment_id, student_id)
# copy files over from the source directory
self.log.info("Overwriting files with master versions from the source directory")
dest_path = self._format_dest(assignment_id, student_id)
source_path = self.directory_structure.format(
nbgrader_step=self.source_directory,
student_id='.',
assignment_id=assignment_id)
source_files = utils.find_all_files(source_path, self.ignore + ["*.ipynb"])
# copy them to the build directory
for filename in source_files:
dest = os.path.join(dest_path, os.path.relpath(filename, source_path))
ensure_dir_exists(os.path.dirname(dest))
if not os.path.normpath(dest) == os.path.normpath(filename):
self.log.info("Linking %s -> %s", filename, dest)
link_or_copy(filename, dest)
# ignore notebooks that aren't in the database
notebooks = []
for notebook in self.notebooks:
notebook_id = os.path.splitext(os.path.basename(notebook))[0]
try:
gb.find_notebook(notebook_id, assignment_id)
except MissingEntry:
self.log.warning("Skipping unknown notebook: %s", notebook)
continue
else:
notebooks.append(notebook)
self.notebooks = notebooks
def run(self):
install.run(self)
from jupyter_client.kernelspec import KernelSpecManager
from IPython.utils.path import ensure_dir_exists
destdir = os.path.join(KernelSpecManager().user_kernel_dir, 'GDL')
ensure_dir_exists(destdir)
with open(os.path.join(destdir, 'kernel.json'), 'w') as f:
json.dump(kernel_json, f, sort_keys=True)
def run(self):
install.run(self)
from IPython.kernel.kernelspec import KernelSpecManager
from IPython.utils.path import ensure_dir_exists
destdir = os.path.join(KernelSpecManager().user_kernel_dir,'IDL')
ensure_dir_exists(destdir)
with open(os.path.join(destdir,'kernel.json'),'w') as f:
json.dump(kernel_json,f,sort_keys=True)
def get_checkpoint_path(self, checkpoint_id, name, path=""):
"""find the path to a checkpoint"""
path = path.strip("/")
basename, ext = os.path.splitext(name)
filename = u"{name}-{checkpoint_id}{ext}".format(name=basename, checkpoint_id=checkpoint_id, ext=ext)
os_path = self._get_os_path(path=path)
cp_dir = os.path.join(os_path, self.checkpoint_dir)
ensure_dir_exists(cp_dir)
cp_path = os.path.join(cp_dir, filename)
return cp_path
def run(self):
# Regular installation
install.run(self)
# Now write the kernelspec
from IPython.kernel.kernelspec import KernelSpecManager
from IPython.utils.path import ensure_dir_exists
destdir = os.path.join(KernelSpecManager().user_kernel_dir, LANGUAGE)
ensure_dir_exists(destdir)
with open(os.path.join(destdir, 'kernel.json'), 'w') as f:
json.dump(kernel_json, f, sort_keys=True)
def run(self):
# Regular installation
install.run(self)
# Now write the kernelspec
from IPython.kernel.kernelspec import KernelSpecManager
from IPython.utils.path import ensure_dir_exists
destdir = os.path.join(KernelSpecManager().user_kernel_dir, 'test')
ensure_dir_exists(destdir)
with open(os.path.join(destdir, 'kernel.json'), 'w') as f:
json.dump(kernel_json, f, sort_keys=True)
def get_ipython_cache_dir():
"""Get the cache directory it is created if it does not exist."""
xdgdir = get_xdg_cache_dir()
if xdgdir is None:
return get_ipython_dir()
ipdir = os.path.join(xdgdir, "ipython")
if not os.path.exists(ipdir) and _writable_dir(xdgdir):
ensure_dir_exists(ipdir)
elif not _writable_dir(xdgdir):
return get_ipython_dir()
return py3compat.cast_unicode(ipdir, fs_encoding)
def get_ipython_cache_dir() -> str:
"""Get the cache directory it is created if it does not exist."""
xdgdir = get_xdg_cache_dir()
if xdgdir is None:
return get_ipython_dir()
ipdir = os.path.join(xdgdir, "ipython")
if not os.path.exists(ipdir) and _writable_dir(xdgdir):
ensure_dir_exists(ipdir)
elif not _writable_dir(xdgdir):
return get_ipython_dir()
return ipdir
def get_ipython_cache_dir():
"""Get the cache directory it is created if it does not exist."""
xdgdir = get_xdg_cache_dir()
if xdgdir is None:
return get_ipython_dir()
ipdir = os.path.join(xdgdir, "ipython")
if not os.path.exists(ipdir) and _writable_dir(xdgdir):
ensure_dir_exists(ipdir)
elif not _writable_dir(xdgdir):
return get_ipython_dir()
return py3compat.cast_unicode(ipdir, fs_encoding)
def _location_changed(self, name, old, new):
if self._location_isset:
raise RuntimeError("Cannot set profile location more than once.")
self._location_isset = True
ensure_dir_exists(new)
# ensure config files exist:
self.security_dir = os.path.join(new, self.security_dir_name)
self.log_dir = os.path.join(new, self.log_dir_name)
self.startup_dir = os.path.join(new, self.startup_dir_name)
self.pid_dir = os.path.join(new, self.pid_dir_name)
self.static_dir = os.path.join(new, self.static_dir_name)
self.check_dirs()
def checkpoint_path(self, checkpoint_id, path):
"""find the path to a checkpoint"""
path = path.strip("/")
parent, name = ("/" + path).rsplit("/", 1)
parent = parent.strip("/")
basename, ext = os.path.splitext(name)
filename = u"{name}-{checkpoint_id}{ext}".format(name=basename, checkpoint_id=checkpoint_id, ext=ext)
os_path = self._get_os_path(path=parent)
cp_dir = os.path.join(os_path, self.checkpoint_dir)
with self.perm_to_403():
ensure_dir_exists(cp_dir)
cp_path = os.path.join(cp_dir, filename)
return cp_path
def apply_filters(params):
if 'EstPresente' in params['Filters']:
#print 'Est_Presente'
Filtered_data, Extracted_data = Estudiante_presente(params)
if 'Copia' in params['Filters']:
Filtered_data, Extracted_data = copia(params)
#print 'Copy'
if 'Omision' in params['Filters']:
Filtered_data, Extracted_data = omision(params)
#print 'Omision'
output_calibracion = params['main_path'].replace('input','output').replace(params['con_file'],'calibracion/'+params['con_file'])
output_calificacion = params['main_path'].replace('input','output').replace(params['con_file'],'calificacion/'+params['con_file'])
ensure_dir_exists(output_calibracion)
ensure_dir_exists(output_calificacion)
def get_checkpoint_path(self, checkpoint_id, name, path=''):
"""find the path to a checkpoint"""
path = path.strip('/')
basename, _ = os.path.splitext(name)
filename = u"{name}-{checkpoint_id}{ext}".format(
name=basename,
checkpoint_id=checkpoint_id,
ext=self.filename_ext,
)
os_path = self._get_os_path(path=path)
cp_dir = os.path.join(os_path, self.checkpoint_dir)
ensure_dir_exists(cp_dir)
cp_path = os.path.join(cp_dir, filename)
return cp_path
def download_checkpoints(db_url, directory, user):
"""
Download users' most recent checkpoints to the given directory.
"""
print("Synchronizing user {user} to {directory}".format(
user=user, directory=directory,
))
ensure_dir_exists(directory)
contents_mgr = FileContentsManager(root_dir=directory)
cp_mgr = PostgresCheckpoints(
db_url=db_url,
user_id=user,
create_user_on_startup=False,
)
cp_mgr.dump(contents_mgr)
print("Done")
def run(self):
# Regular installation
install.run(self)
# Now write the kernelspec
try:
from jupyter_client.kernelspec import KernelSpecManager
except ImportError:
from IPython.kernel.kernelspec import KernelSpecManager
# from IPython.kernel.kernelspec import KernelSpecManager
from IPython.utils.path import ensure_dir_exists
destdir = os.path.join(KernelSpecManager().user_kernel_dir, "OpenModelica")
ensure_dir_exists(destdir)
with open(os.path.join(destdir, 'kernel.json'), 'w') as f:
json.dump(kernel_json, f, sort_keys=True)
def get_checkpoint_path(self, checkpoint_id, path):
"""find the path to a checkpoint"""
path = path.strip('/')
parent, name = ('/' + path).rsplit('/', 1)
parent = parent.strip('/')
basename, ext = os.path.splitext(name)
filename = u"{name}-{checkpoint_id}{ext}".format(
name=basename,
checkpoint_id=checkpoint_id,
ext=ext,
)
os_path = self._get_os_path(path=parent)
cp_dir = os.path.join(os_path, self.checkpoint_dir)
ensure_dir_exists(cp_dir)
cp_path = os.path.join(cp_dir, filename)
return cp_path
def get_checkpoint_path(self, checkpoint_id, path):
"""find the path to a checkpoint"""
path = path.strip('/')
parent, name = ('/' + path).rsplit('/', 1)
parent = parent.strip('/')
basename, ext = os.path.splitext(name)
filename = u"{name}-{checkpoint_id}{ext}".format(
name=basename,
checkpoint_id=checkpoint_id,
ext=ext,
)
os_path = self._get_os_path(path=parent)
cp_dir = os.path.join(os_path, self.checkpoint_dir)
with self.perm_to_403():
ensure_dir_exists(cp_dir)
cp_path = os.path.join(cp_dir, filename)
return cp_path
def run(self):
#Regular installation
install.run(self)
#Now write the kernelspec
try:
from jupyter_client.kernelspec import KernelSpecManager
except ImportError:
from IPython.kernel.kernelspec import KernelSpecManager
#from IPython.kernel.kernelspec import KernelSpecManager
from IPython.utils.path import ensure_dir_exists
destdir = os.path.join(KernelSpecManager().user_kernel_dir,
"OpenModelica")
ensure_dir_exists(destdir)
with open(os.path.join(destdir, 'kernel.json'), 'w') as f:
json.dump(kernel_json, f, sort_keys=True)
def download_checkpoints(db_url, directory, user):
"""
Download users' most recent checkpoints to the given directory.
"""
print("Synchronizing user {user} to {directory}".format(
user=user,
directory=directory,
))
ensure_dir_exists(directory)
contents_mgr = FileContentsManager(root_dir=directory)
cp_mgr = PostgresCheckpoints(
db_url=db_url,
user_id=user,
create_user_on_startup=False,
)
cp_mgr.dump(contents_mgr)
print("Done")
def init_assignment(self, assignment_id, student_id):
"""Initializes resources/dependencies/etc. that are common to all
notebooks in an assignment.
"""
source = self._format_source(assignment_id, student_id)
dest = self._format_dest(assignment_id, student_id)
# detect other files in the source directory
for filename in find_all_files(source, self.ignore + ["*.ipynb"]):
# Make sure folder exists.
path = os.path.join(dest, os.path.relpath(filename, source))
ensure_dir_exists(os.path.dirname(path))
# Copy if destination is different.
if not os.path.normpath(path) == os.path.normpath(filename):
self.log.info("Linking %s -> %s", filename, path)
link_or_copy(filename, path)
def init_assignment(self, assignment_id, student_id):
"""Initializes resources/dependencies/etc. that are common to all
notebooks in an assignment.
"""
source = self._format_source(assignment_id, student_id)
dest = self._format_dest(assignment_id, student_id)
# detect other files in the source directory
for filename in find_all_files(source, self.ignore + ["*.ipynb"]):
# Make sure folder exists.
path = os.path.join(dest, os.path.relpath(filename, source))
ensure_dir_exists(os.path.dirname(path))
# Copy if destination is different.
if not os.path.normpath(path) == os.path.normpath(filename):
self.log.info("Linking %s -> %s", filename, path)
link_or_copy(filename, path)
def subdivide_Parquet(source='', destination='', x=2, y=2):
"""Utility function to subdivide the RGB images from the Parquet database.
Parameters
----------
source : string
Full path of the folder where the original RGB images are stored.
destination : string
Full path of the folder where the subimages will be saved.
x : int
Number of vertical subdivisions.
y : int
Number of horizontal subdivisions.
"""
#source = r'D:\mydatadrive\Datos\Texture datasets\Texture databases\Parquet'
#destination = r'C:\texture\images\Parquet'
from skimage import io
from IPython.utils.path import ensure_dir_exists
original = [os.path.join(root, filename)
for root, dirs, files in os.walk(source)
for filename in files
if imghdr.what(os.path.join(root, filename))]
ensure_dir_exists(destination)
for o in original:
img = io.imread(o)
_, file_source = os.path.split(o)
name, ext = os.path.splitext(file_source)
klass, sample = name.split('__')
ensure_dir_exists(os.path.join(destination, klass))
rows = np.int_(np.linspace(0, img.shape[0], x + 1))
cols = np.int_(np.linspace(0, img.shape[1], y + 1))
for i, (start_row, end_row) in enumerate(zip(rows[:-1], rows[1:])):
for j, (start_col, end_col) in enumerate(zip(cols[:-1], cols[1:])):
sub = img[start_row:end_row, start_col:end_col, :]
sample = 1 + i*y + j
subname = f'{name}_{sample}{ext}'
subpath = os.path.join(destination, klass, subname)
io.imsave(subpath, sub)
def _ipython_dir_changed(self, change):
old = change['old']
new = change['new']
if old is not Undefined:
str_old = os.path.abspath(old)
if str_old in sys.path:
sys.path.remove(str_old)
str_path = os.path.abspath(new)
sys.path.append(str_path)
ensure_dir_exists(new)
readme = os.path.join(new, 'README')
readme_src = os.path.join(get_ipython_package_dir(), u'config', u'profile', 'README')
if not os.path.exists(readme) and os.path.exists(readme_src):
shutil.copy(readme_src, readme)
for d in ('extensions', 'nbextensions'):
path = os.path.join(new, d)
try:
ensure_dir_exists(path)
except OSError as e:
# this will not be EEXIST
self.log.error("couldn't create path %s: %s", path, e)
self.log.debug("IPYTHONDIR set to: %s" % new)
def _ipython_dir_changed(self, change):
old = change["old"]
new = change["new"]
if old is not Undefined:
str_old = py3compat.cast_bytes_py2(os.path.abspath(old), sys.getfilesystemencoding())
if str_old in sys.path:
sys.path.remove(str_old)
str_path = py3compat.cast_bytes_py2(os.path.abspath(new), sys.getfilesystemencoding())
sys.path.append(str_path)
ensure_dir_exists(new)
readme = os.path.join(new, "README")
readme_src = os.path.join(get_ipython_package_dir(), u"config", u"profile", "README")
if not os.path.exists(readme) and os.path.exists(readme_src):
shutil.copy(readme_src, readme)
for d in ("extensions", "nbextensions"):
path = os.path.join(new, d)
try:
ensure_dir_exists(path)
except OSError as e:
# this will not be EEXIST
self.log.error("couldn't create path %s: %s", path, e)
self.log.debug("IPYTHONDIR set to: %s" % new)
def extract_package(name, client=xc.ServerProxy('https://pypi.python.org/pypi')):
try:
release = client.package_releases(name)[-1]
outdir = 'pack_info/{}/'.format(name)
doc = client.release_urls(name, release)
if doc:
url = None
for d in doc:
if d['python_version'] == 'source' and d['url'].endswith('gz'):
url = d['url']
if url:
req = requests.get(url)
if req.status_code != 200:
print("Could not download file %s" % req.status_code)
else:
#print(outdir)
ensure_dir_exists('{}'.format(outdir))
with open('/tmp/temp_tar', 'wb') as tar_file:
tar_file.write(req.content)
with open('/tmp/temp_tar', 'rb') as tar_file:
return _extract_files(tar_file, outdir, name)
except:
pass
def test_ensure_dir_exists():
with TemporaryDirectory() as td:
d = os.path.join(td, '∂ir')
path.ensure_dir_exists(d) # create it
assert os.path.isdir(d)
path.ensure_dir_exists(d) # no-op
f = os.path.join(td, "ƒile")
open(f, "w", encoding="utf-8").close() # touch
with pytest.raises(IOError):
path.ensure_dir_exists(f)
def test_ensure_dir_exists():
with TemporaryDirectory() as td:
d = os.path.join(td, '∂ir')
path.ensure_dir_exists(d) # create it
assert os.path.isdir(d)
path.ensure_dir_exists(d) # no-op
f = os.path.join(td, 'ƒile')
open(f, 'w').close() # touch
with nt.assert_raises(IOError):
path.ensure_dir_exists(f)
def test_ensure_dir_exists():
with TemporaryDirectory() as td:
d = os.path.join(td, '∂ir')
path.ensure_dir_exists(d) # create it
assert os.path.isdir(d)
path.ensure_dir_exists(d) # no-op
f = os.path.join(td, 'ƒile')
open(f, 'w').close() # touch
with nt.assert_raises(IOError):
path.ensure_dir_exists(f)
def _get_image_tag(self, match, path = None, format = "png"):
""" Return (X)HTML mark-up for the image-tag given by match.
Parameters
----------
match : re.SRE_Match
A match to an HTML image tag as exported by Qt, with
match.group("Name") containing the matched image ID.
path : string|None, optional [default None]
If not None, specifies a path to which supporting files may be
written (e.g., for linked images). If None, all images are to be
included inline.
format : "png"|"svg"|"jpg", optional [default "png"]
Format for returned or referenced images.
"""
if format in ("png","jpg"):
try:
image = self._get_image(match.group("name"))
except KeyError:
return "<b>Couldn't find image %s</b>" % match.group("name")
if path is not None:
ensure_dir_exists(path)
relpath = os.path.basename(path)
if image.save("%s/qt_img%s.%s" % (path, match.group("name"), format),
"PNG"):
return '<img src="%s/qt_img%s.%s">' % (relpath,
match.group("name"),format)
else:
return "<b>Couldn't save image!</b>"
else:
ba = QtCore.QByteArray()
buffer_ = QtCore.QBuffer(ba)
buffer_.open(QtCore.QIODevice.WriteOnly)
image.save(buffer_, format.upper())
buffer_.close()
return '<img src="data:image/%s;base64,\n%s\n" />' % (
format,re.sub(r'(.{60})',r'\1\n',str(ba.toBase64())))
elif format == "svg":
try:
svg = str(self._name_to_svg_map[match.group("name")])
except KeyError:
if not self._svg_warning_displayed:
QtGui.QMessageBox.warning(self, 'Error converting PNG to SVG.',
'Cannot convert PNG images to SVG, export with PNG figures instead. '
'If you want to export matplotlib figures as SVG, add '
'to your ipython config:\n\n'
'\tc.InlineBackend.figure_format = \'svg\'\n\n'
'And regenerate the figures.',
QtGui.QMessageBox.Ok)
self._svg_warning_displayed = True
return ("<b>Cannot convert PNG images to SVG.</b> "
"You must export this session with PNG images. "
"If you want to export matplotlib figures as SVG, add to your config "
"<span>c.InlineBackend.figure_format = 'svg'</span> "
"and regenerate the figures.")
# Not currently checking path, because it's tricky to find a
# cross-browser way to embed external SVG images (e.g., via
# object or embed tags).
# Chop stand-alone header from matplotlib SVG
offset = svg.find("<svg")
assert(offset > -1)
return svg[offset:]
else:
return '<b>Unrecognized image format</b>'
def install_nbextension(files, overwrite=False, symlink=False, user=False, prefix=None, nbextensions_dir=None, verbose=1):
"""Install a Javascript extension for the notebook
Stages files and/or directories into the nbextensions directory.
By default, this compares modification time, and only stages files that need updating.
If `overwrite` is specified, matching files are purged before proceeding.
Parameters
----------
files : list(paths or URLs) or dict(install_name: path or URL)
One or more paths or URLs to existing files directories to install.
If given as a list, these will be installed with their base name, so '/path/to/foo'
will install to 'nbextensions/foo'. If given as a dict, such as {'bar': '/path/to/foo'},
then '/path/to/foo' will install to 'nbextensions/bar'.
Archives (zip or tarballs) will be extracted into the nbextensions directory.
overwrite : bool [default: False]
If True, always install the files, regardless of what may already be installed.
symlink : bool [default: False]
If True, create a symlink in nbextensions, rather than copying files.
Not allowed with URLs or archives. Windows support for symlinks requires
Vista or above, Python 3, and a permission bit which only admin users
have by default, so don't rely on it.
user : bool [default: False]
Whether to install to the user's .ipython/nbextensions directory.
Otherwise do a system-wide install (e.g. /usr/local/share/jupyter/nbextensions).
prefix : str [optional]
Specify install prefix, if it should differ from default (e.g. /usr/local).
Will install to prefix/share/jupyter/nbextensions
nbextensions_dir : str [optional]
Specify absolute path of nbextensions directory explicitly.
verbose : int [default: 1]
Set verbosity level. The default is 1, where file actions are printed.
set verbose=2 for more output, or verbose=0 for silence.
"""
nbext = _get_nbext_dir(nbextensions_dir, user, prefix)
# make sure nbextensions dir exists
ensure_dir_exists(nbext)
if isinstance(files, string_types):
# one file given, turn it into a list
files = [files]
if isinstance(files, (list,tuple)):
# list given, turn into dict
_files = {}
for path in map(cast_unicode_py2, files):
if path.startswith(('https://', 'http://')):
destination = urlparse(path).path.split('/')[-1]
elif path.endswith('.zip') or _safe_is_tarfile(path):
destination = str(uuid.uuid4()) # ignored for archives
else:
destination = basename(path)
_files[destination] = path
files = _files
for dest_basename,path in (map(cast_unicode_py2, item) for item in files.items()):
if path.startswith(('https://', 'http://')):
if symlink:
raise ValueError("Cannot symlink from URLs")
# Given a URL, download it
with TemporaryDirectory() as td:
filename = urlparse(path).path.split('/')[-1]
local_path = os.path.join(td, filename)
if verbose >= 1:
print("downloading %s to %s" % (path, local_path))
urlretrieve(path, local_path)
# now install from the local copy
install_nbextension({dest_basename: local_path}, overwrite=overwrite, symlink=symlink, nbextensions_dir=nbext, verbose=verbose)
continue
# handle archives
archive = None
if path.endswith('.zip'):
archive = zipfile.ZipFile(path)
elif _safe_is_tarfile(path):
archive = tarfile.open(path)
if archive:
if symlink:
raise ValueError("Cannot symlink from archives")
if verbose >= 1:
print("extracting %s to %s" % (path, nbext))
archive.extractall(nbext)
archive.close()
continue
dest = pjoin(nbext, dest_basename)
if overwrite and os.path.exists(dest):
if verbose >= 1:
print("removing %s" % dest)
if os.path.isdir(dest) and not os.path.islink(dest):
shutil.rmtree(dest)
else:
os.remove(dest)
if symlink:
path = os.path.abspath(path)
if not os.path.exists(dest):
if verbose >= 1:
print("symlink %s -> %s" % (dest, path))
os.symlink(path, dest)
continue
if os.path.isdir(path):
path = pjoin(os.path.abspath(path), '') # end in path separator
for parent, dirs, files in os.walk(path):
dest_dir = pjoin(dest, parent[len(path):])
if not os.path.exists(dest_dir):
if verbose >= 2:
print("making directory %s" % dest_dir)
os.makedirs(dest_dir)
for file in files:
src = pjoin(parent, file)
# print("%r, %r" % (dest_dir, file))
dest_file = pjoin(dest_dir, file)
_maybe_copy(src, dest_file, verbose)
else:
src = path
_maybe_copy(src, dest, verbose)
def main():
try:
if not os.path.exists(sys.argv[1]):
sys.exit("O_O No existe el archivo de parametros: ./" +
sys.argv[1])
with open(sys.argv[1], 'r') as f:
dic = json.load(f)
f.close()
dic['main_dir'] = os.getcwd().replace('\\', '/')
dic['copy_file'] = os.getcwd().replace('\\',
'/') + '/' + dic['copy_file']
dic['Filters'] = ['EstPresente', 'Copia', 'Omision']
# # Archivo de logging
if os.path.exists(dic['main_dir'] + '/preprocessing.log'):
os.remove(dic['main_dir'] + '/preprocessing.log')
logging.basicConfig(
filename=dic['main_dir'] + '/preprocessing.log',
level=logging.INFO,
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p')
# validar archivo de SNP actualizacion
if len(sys.argv) == 3:
flagUpdate = True
if not os.path.exists(dic['main_dir'] + '/input/' + sys.argv[2]):
logging.info("ADVERTENCIA .... No existe el archivo (" +
sys.argv[2] + ") No se filtraran los .dat")
else:
pdSNP = pd.read_csv(
dic['main_dir'] + '/input/' + sys.argv[2], dtype="str")
else:
flagUpdate = False
print('REVISANDO PARAMETROS...')
if not os.path.exists(dic['main_dir'] + '/src/'):
sys.exit("FALTA CARPETA SRC")
if not os.path.exists(dic['main_dir'] + '/src/bin/'):
sys.exit(
"CARPETA BIN PARA BILOG NO EXISTE, COPIAR CARPETA EN 'src/'")
if not os.path.exists(dic['main_dir'] + '/input'):
sys.exit(
"CARPETA 'input' NO EXISTE --> CREAR CARPETA INPUT CON .zip Y ARCHIVO DE COPIA"
)
if not os.path.isfile(dic['copy_file']):
sys.exit(
"ARCHIVO DE COPIA NO ENCONTRADO --> REVISAR ARCHIVO DE PARAMETROS 'src/parameters.json' --> parametro 'copy_file'"
)
isMissing = [
prueba for prueba in dic['Pruebas'].keys()
if not prueba in dic['Codigos'].keys()
]
if len(isMissing) > 0:
sys.exit(
"PARAMETROS INCORRECTOS DE JUNTURAS FALTA AGREGAR: ---->\n" +
', '.join(isMissing) +
"\n-------------------------------------------------------" +
"\ncambiar en 'src/parameters.json' --> parametro 'Codigos'")
isMissing = [
prueba for prueba in dic['Codigos'].keys()
if not prueba in dic['Pruebas'].keys()
]
if len(isMissing) > 0:
sys.exit(
"PARAMETROS INCORRECTOS DE FORMAS FALTA AGREGAR: ---->\n" +
', '.join(isMissing) +
"\n-------------------------------------------------------" +
"\ncambiar en 'src/parameters.json' --> parametro 'Codigos'")
logging.info(
'#######################################################################'
)
logging.info('COMENZO: ' +
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
logging.info('Corriendo Preprocessing_stage.py')
logging.info(
'#######################################################################'
)
if dic['subloque']:
if ("subloques" not in dic.keys()):
logging.info(
'NO DEFINIO SUBLOQUES, no se correran sub-bloques...')
dic['subloques'] = []
dic['subloque'] = False
if len(dic['subloques']) == 0:
logging.info(
'NO DEFINIO SUBLOQUES, no se correran sub-bloques...')
dic['subloque'] = False
else:
logging.info(
'NO DEFINIO BANDERA SUBLOQUE, no se correran sub-bloques...')
logging.info('Limpiando carpeta principal')
if os.path.exists(dic['main_dir'] + '/input/Descargas'):
shutil.rmtree(
dic['main_dir'] + '/input/Descargas', ignore_errors=True)
if os.path.exists(dic['main_dir'] + '/output'):
shutil.rmtree(dic['main_dir'] + '/output', ignore_errors=True)
if os.path.exists(dic['main_dir'] + '/doc'):
shutil.rmtree(dic['main_dir'] + '/doc', ignore_errors=True)
time.sleep(15)
logging.info('\tLeyendo archivo de parametros: ' + sys.argv[1])
zipfiles = []
logging.info(
"ARCHIVOS .ZIP ENCONTRADOS en 'input'--> ALGUNA INCONSISTENCIA EN LOS ARCHIVOS --> REVISAR CARPETA INPUT Y EJECUTAR NUEVAMENTE EL PROCESO"
)
for file in os.listdir(dic['main_dir'] + '/input/'):
if file.endswith('.zip'):
zipfiles.append(file)
if len(zipfiles) == 0:
sys.exit(
"NINGUN ARCHIVO ZIP ENCONTRADO -- > COPIAR ARCHIVOS ZIP EN 'input/'"
)
else:
logging.info(str(zipfiles))
logging.info(sep)
logging.info('REVISANDO SCRIPTS NECESARIOS...')
dirs = os.listdir(dic['main_dir'] + '/src')
if not 'JuntarModulosSaber11.pl' in dirs:
logging.info("FALTA SCRIPT 'src/JuntarModulosSaber11.pl'")
sys.exit("FALTA SCRIPT 'src/JuntarModulosSaber11.pl'")
groups = {
'01_Estudiantes': None,
'02_NoEstudiantes': None,
'03_Discapacitados': None
}
folder_structure = {
'input': None,
'output': {
'calibracion': groups,
'calificacion': groups
},
'doc': None
}
logging.info(sep)
logging.info('CREANDO ESTRUCTURA DE CARPETAS EN: ' + dic['main_dir'] +
'/' + str(folder_structure) + '...')
utils.make_dirs_from_dict(folder_structure, dic['main_dir'] + '/')
logging.info('\tTerminado...')
logging.info(sep)
logging.info('EXTRAYENDO ARCHIVOS...')
utils.extract_files(dic['main_dir'] + '/')
logging.info('\tTerminado...')
logging.info(sep)
if dic['subloque']:
logging.info('ELIMINANDO SUBLOQUES 0...')
for root, dirs, files in os.walk(dic["main_dir"]):
for file in files:
pattern = re.compile(r".*sblq(.*)\.|\-.*", re.I)
matcher = pattern.match(file)
if not matcher == None and not matcher.groups()[0].split(
'-')[0] in dic['subloques']:
print('ARCHIVO A ELIMINAR..', file)
os.remove(os.path.join(root, file))
logging.info('\tTerminado...')
logging.info(sep)
logging.info('GENERANDO FORMAS DE SUBLOQUES...')
print('Generando formas de subloques')
pattern = re.compile(r'.*sblq(.*).con', re.I)
ext = dic['out_f'][:]
ext.remove('.con')
for root, dirs, files in os.walk(dic["main_dir"]):
for file in files:
matcher = pattern.search(file)
if not matcher == None:
newPath = root + '_' + matcher.groups()[0]
if not os.path.exists(newPath):
print('NEW_FOLDER: ' + newPath)
ensure_dir_exists(newPath + '/salidas')
params = {
'main_path': root + '/',
'con_file': file.replace('.con', '')
}
utils.filterISELECT(params, newPath)
logging.info('\tTerminado...')
logging.info(sep)
logging.info('ADICIONANDO SUBLOQUES A DICCIONARIO...')
print('Adicionando subloques a diccionario...')
utils.add_subloques(dic)
logging.info(dic['Pruebas'])
logging.info('\tTerminado...')
logging.info(sep)
logging.info('ARCHIVOS DE SUBLOQUES...')
print('Eliminando archivos de subloques')
for root, dirs, files in os.walk(dic["main_dir"]):
for file in files:
if re.match("(.*)sblq(.*)", file):
print('ARCHIVO A ELIMINAR..', file)
os.remove(os.path.join(root, file))
logging.info('\tTerminado...')
logging.info(sep)
logging.info('APLICANDO FILTROS A ARCHIVOS .DAT...')
mpath = ''
for root, dirs, files in os.walk(dic["main_dir"]):
for file in files:
if file.endswith(".con"):
flagFiltro = True
print('CONFILE ENCONTRADO: ' + file)
path = (os.path.join(root, file)).replace('\\',
'/').split('/')
mpath = ('/').join((os.path.join(root, file)).replace(
'\\', '/').split('/')[:-1]) + '/'
if dic['subloque']:
indSubl = [
bloque in file for bloque in dic['subloques']
]
print(indSubl)
if not any(indSubl):
flagFiltro = False
shutil.rmtree(root)
if flagFiltro:
confile = path[-2]
dic['con_file'] = path[-2]
logging.info('\tAplicando filtros a : ' + confile)
dic['filtered_data'] = ''
dic['main_path'] = mpath
dic['count_log'] = dic['main_path'] + '/registro.log'
con = utils.create_dict_from_con(dic)
params = dict(dic.items() + con.items())
params['id_form'] = confile
params['aplicacion'] = con['DATA'].split('-')[0][0:7]
logging.info('\t\tAplicacion: ' + params['aplicacion'])
f_g = {}
utils.set_f_g(params['Pruebas'].copy(), confile, f_g)
if not f_g == {}:
params['curr_group'] = f_g[confile]
logging.info('\t\tGrupo: ' + params['curr_group'])
if not flagUpdate:
filtrado.apply_filters(params)
mpath = mpath.replace(confile, '')
logging.info('\t\tTerminado...')
logging.info(sep)
if not flagUpdate:
logging.info('CREANDO ARCHIVO DE CONFIGURACION.TXT...')
f = []
for root, dirs, files in os.walk(dic['main_dir'] +
'/input/Descargas/'):
for dir in dirs:
f.append(dir)
config_file = []
utils.create_config_file(dic['Pruebas'], dic['Codigos'],
dic['Pruebas'].keys(), '', config_file,
[], f)
np.savetxt(
dic['main_dir'] + '/output/configuracion.txt',
config_file,
delimiter=",",
fmt="%s")
logging.info('\t\tTerminado...')
logging.info(sep)
logging.info('JUNTANDO ARCHIVOS DAT (JUNTAR.pl)...')
os.chdir(mpath) #Change to forms_path
p = subprocess.Popen([
'perl', dic['main_dir'] + '/src/JuntarModulosSaber11.pl',
'-com', '-dat', '-conf',
dic['main_dir'] + '/output/configuracion.txt'
])
p.communicate()
logging.info('\t\tTerminado...')
logging.info(sep)
ext = dic['out_f']
logging.info(
'MOVIENDO ARCHIVOS A SUS RESPECTIVAS CARPETAS DE SALIDA...')
config = []
logging.info(
'SACANDO CARPETAS DE JUNTURA...(Leyendo archivo de configuracion.txt)'
)
with open(dic['main_dir'] + '/output/configuracion.txt') as f:
config = f.readlines()
f.close()
J = [] #Formas ya presentes en carpeta JUNTAS
for line in config:
if line.startswith("PRUEBA"):
J.append(line.split()[2])
logging.info('FORMAS DE JUNTURA: ' + str(J))
dirs = os.listdir(os.getcwd())
for d in dirs:
if not d in J and os.path.isdir(os.getcwd() + '/' + d):
logging.info(sep)
logging.info('PATH DE INPUT - OUTPUT PARA CARPETA: ' + d)
if d == "JUNTAS":
f_g = {'JUNTAS': '01_Estudiantes'}
else:
f_g = {}
utils.set_f_g(dic['Pruebas'].copy(), d, f_g)
if not f_g == {}:
path_output = dic[
'main_dir'] + '/output/calibracion/' + f_g[
d] + '/' + d
ensure_dir_exists(path_output)
if d == 'JUNTAS':
path_input = os.getcwd() + '/' + d
logging.info('PATH INPUT: ' + path_input)
logging.info('PATH OUTPUT: ' + path_output)
utils.copytree(path_input, path_output)
else:
for root, dirs, files in os.walk(os.getcwd() +
'/' + d):
for file in files:
for ex in dic['out_f']:
if ex in file:
path_input = os.path.join(
root, file).replace('\\', '/')
logging.info('PATH INPUT: ' +
path_input)
logging.info('PATH OUTPUT: ' +
path_output + '/' +
file)
shutil.copyfile(
path_input,
path_output + '/' + file)
if not d == 'JUNTAS':
ensure_dir_exists(path_output + '/salidas')
logging.info('SALIDAS: ' + path_output + '/salidas')
logging.info(sep)
logging.info(
'MOVIENDO ARCHIVOS DE CALIFICACION A SUS RESPECTIVAS CARPETAS...')
ext = []
for e in dic['out_f']:
if e.endswith('.DAT') and not flagUpdate:
ext.append(e.replace('.DAT', '.O'))
else:
ext.append(e)
removePaths = []
for root, dirs, files in os.walk(os.getcwd()):
for d in dirs:
if not d == 'JUNTAS':
f_g = {}
utils.set_f_g(dic['Pruebas'].copy(), d, f_g)
if not f_g == {}:
path_output = dic[
'main_dir'] + '/output/calificacion/' + f_g[
d] + '/' + d
ensure_dir_exists(path_output)
if not os.path.exists(path_output + '/salidas'):
os.makedirs(path_output + '/salidas')
for file in os.listdir(os.path.join(root, d)):
for e in ext:
if e in file:
path_input = os.path.join(root,
d) + '/' + file
if file.endswith('.O'):
output = path_output + '/' + file.replace(
'.O', '.DAT')
else:
output = path_output + '/' + file
if not os.path.exists(output):
shutil.copyfile(path_input, output)
if flagUpdate and file.endswith(
'.DAT'):
if ('pdSNP' in locals()):
nUpdate = filtroActualizacion(
output, pdSNP,
dic['id_len'])
if nUpdate == 0:
removePaths.append(
path_output)
logging.info(
'FILTRANDO NUEVAS PERSONAS PARA CALIFICAR: '
+ file)
else:
statFile = os.stat(path_input)
if str(statFile.
st_size) == '0':
removePaths.append(
path_output)
logging.info(
'SE CALIFICARAN TODAS LAS PERSONAS: '
+ file)
# # Removiendo
if flagUpdate:
logging.info(sep)
logging.info('ELIMANDO CARPETAS EN BLANCO DE LA CALIFICACION...')
for path in removePaths:
logging.info('Eliminando: ' + path)
shutil.rmtree(path)
logging.info(sep)
logging.info('\tTerminado...')
logging.info(sep)
logging.info('TERMINO: ' +
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
except Exception as e:
print(logging.error(traceback.format_exc()))
def install_nbextension(path, overwrite=False, symlink=False, user=False, prefix=None, nbextensions_dir=None, destination=None, verbose=1):
"""Install a Javascript extension for the notebook
Stages files and/or directories into the nbextensions directory.
By default, this compares modification time, and only stages files that need updating.
If `overwrite` is specified, matching files are purged before proceeding.
Parameters
----------
path : path to file, directory, zip or tarball archive, or URL to install
By default, the file will be installed with its base name, so '/path/to/foo'
will install to 'nbextensions/foo'. See the destination argument below to change this.
Archives (zip or tarballs) will be extracted into the nbextensions directory.
overwrite : bool [default: False]
If True, always install the files, regardless of what may already be installed.
symlink : bool [default: False]
If True, create a symlink in nbextensions, rather than copying files.
Not allowed with URLs or archives. Windows support for symlinks requires
Vista or above, Python 3, and a permission bit which only admin users
have by default, so don't rely on it.
user : bool [default: False]
Whether to install to the user's .ipython/nbextensions directory.
Otherwise do a system-wide install (e.g. /usr/local/share/jupyter/nbextensions).
prefix : str [optional]
Specify install prefix, if it should differ from default (e.g. /usr/local).
Will install to ``<prefix>/share/jupyter/nbextensions``
nbextensions_dir : str [optional]
Specify absolute path of nbextensions directory explicitly.
destination : str [optional]
name the nbextension is installed to. For example, if destination is 'foo', then
the source file will be installed to 'nbextensions/foo', regardless of the source name.
This cannot be specified if an archive is given as the source.
verbose : int [default: 1]
Set verbosity level. The default is 1, where file actions are printed.
set verbose=2 for more output, or verbose=0 for silence.
"""
nbext = _get_nbext_dir(nbextensions_dir, user, prefix)
# make sure nbextensions dir exists
ensure_dir_exists(nbext)
if isinstance(path, (list, tuple)):
raise TypeError("path must be a string pointing to a single extension to install; call this function multiple times to install multiple extensions")
path = cast_unicode_py2(path)
if path.startswith(('https://', 'http://')):
if symlink:
raise ValueError("Cannot symlink from URLs")
# Given a URL, download it
with TemporaryDirectory() as td:
filename = urlparse(path).path.split('/')[-1]
local_path = os.path.join(td, filename)
if verbose >= 1:
print("downloading %s to %s" % (path, local_path))
urlretrieve(path, local_path)
# now install from the local copy
install_nbextension(local_path, overwrite=overwrite, symlink=symlink, nbextensions_dir=nbext, destination=destination, verbose=verbose)
elif path.endswith('.zip') or _safe_is_tarfile(path):
if symlink:
raise ValueError("Cannot symlink from archives")
if destination:
raise ValueError("Cannot give destination for archives")
if verbose >= 1:
print("extracting %s to %s" % (path, nbext))
if path.endswith('.zip'):
archive = zipfile.ZipFile(path)
elif _safe_is_tarfile(path):
archive = tarfile.open(path)
archive.extractall(nbext)
archive.close()
else:
if not destination:
destination = basename(path)
destination = cast_unicode_py2(destination)
full_dest = pjoin(nbext, destination)
if overwrite and os.path.lexists(full_dest):
if verbose >= 1:
print("removing %s" % full_dest)
if os.path.isdir(full_dest) and not os.path.islink(full_dest):
shutil.rmtree(full_dest)
else:
os.remove(full_dest)
if symlink:
path = os.path.abspath(path)
if not os.path.exists(full_dest):
if verbose >= 1:
print("symlink %s -> %s" % (full_dest, path))
os.symlink(path, full_dest)
elif os.path.isdir(path):
path = pjoin(os.path.abspath(path), '') # end in path separator
for parent, dirs, files in os.walk(path):
dest_dir = pjoin(full_dest, parent[len(path):])
if not os.path.exists(dest_dir):
if verbose >= 2:
print("making directory %s" % dest_dir)
os.makedirs(dest_dir)
for file in files:
src = pjoin(parent, file)
# print("%r, %r" % (dest_dir, file))
dest_file = pjoin(dest_dir, file)
_maybe_copy(src, dest_file, verbose)
else:
src = path
_maybe_copy(src, full_dest, verbose)
def _build_directory_changed(self, name, old, new):
if new:
ensure_dir_exists(new)
def _makedir(self, path):
"""Make a directory if it doesn't already exist"""
if path:
self.log.info("Making directory %s", path)
ensure_dir_exists(path)
def install_nbextension(path,
overwrite=False,
symlink=False,
user=False,
prefix=None,
nbextensions_dir=None,
destination=None,
verbose=1):
"""Install a Javascript extension for the notebook
Stages files and/or directories into the nbextensions directory.
By default, this compares modification time, and only stages files that need updating.
If `overwrite` is specified, matching files are purged before proceeding.
Parameters
----------
path : path to file, directory, zip or tarball archive, or URL to install
By default, the file will be installed with its base name, so '/path/to/foo'
will install to 'nbextensions/foo'. See the destination argument below to change this.
Archives (zip or tarballs) will be extracted into the nbextensions directory.
overwrite : bool [default: False]
If True, always install the files, regardless of what may already be installed.
symlink : bool [default: False]
If True, create a symlink in nbextensions, rather than copying files.
Not allowed with URLs or archives. Windows support for symlinks requires
Vista or above, Python 3, and a permission bit which only admin users
have by default, so don't rely on it.
user : bool [default: False]
Whether to install to the user's .ipython/nbextensions directory.
Otherwise do a system-wide install (e.g. /usr/local/share/jupyter/nbextensions).
prefix : str [optional]
Specify install prefix, if it should differ from default (e.g. /usr/local).
Will install to prefix/share/jupyter/nbextensions
nbextensions_dir : str [optional]
Specify absolute path of nbextensions directory explicitly.
destination : str [optional]
name the nbextension is installed to. For example, if destination is 'foo', then
the source file will be installed to 'nbextensions/foo', regardless of the source name.
This cannot be specified if an archive is given as the source.
verbose : int [default: 1]
Set verbosity level. The default is 1, where file actions are printed.
set verbose=2 for more output, or verbose=0 for silence.
"""
nbext = _get_nbext_dir(nbextensions_dir, user, prefix)
# make sure nbextensions dir exists
ensure_dir_exists(nbext)
if isinstance(path, (list, tuple)):
raise TypeError(
"path must be a string pointing to a single extension to install; call this function multiple times to install multiple extensions"
)
path = cast_unicode_py2(path)
if path.startswith(('https://', 'http://')):
if symlink:
raise ValueError("Cannot symlink from URLs")
# Given a URL, download it
with TemporaryDirectory() as td:
filename = urlparse(path).path.split('/')[-1]
local_path = os.path.join(td, filename)
if verbose >= 1:
print("downloading %s to %s" % (path, local_path))
urlretrieve(path, local_path)
# now install from the local copy
install_nbextension(local_path,
overwrite=overwrite,
symlink=symlink,
nbextensions_dir=nbext,
destination=destination,
verbose=verbose)
elif path.endswith('.zip') or _safe_is_tarfile(path):
if symlink:
raise ValueError("Cannot symlink from archives")
if destination:
raise ValueError("Cannot give destination for archives")
if verbose >= 1:
print("extracting %s to %s" % (path, nbext))
if path.endswith('.zip'):
archive = zipfile.ZipFile(path)
elif _safe_is_tarfile(path):
archive = tarfile.open(path)
archive.extractall(nbext)
archive.close()
else:
if not destination:
destination = basename(path)
destination = cast_unicode_py2(destination)
full_dest = pjoin(nbext, destination)
if overwrite and os.path.lexists(full_dest):
if verbose >= 1:
print("removing %s" % full_dest)
if os.path.isdir(full_dest) and not os.path.islink(full_dest):
shutil.rmtree(full_dest)
else:
os.remove(full_dest)
if symlink:
path = os.path.abspath(path)
if not os.path.exists(full_dest):
if verbose >= 1:
print("symlink %s -> %s" % (full_dest, path))
os.symlink(path, full_dest)
elif os.path.isdir(path):
path = pjoin(os.path.abspath(path), '') # end in path separator
for parent, dirs, files in os.walk(path):
dest_dir = pjoin(full_dest, parent[len(path):])
if not os.path.exists(dest_dir):
if verbose >= 2:
print("making directory %s" % dest_dir)
os.makedirs(dest_dir)
for file in files:
src = pjoin(parent, file)
# print("%r, %r" % (dest_dir, file))
dest_file = pjoin(dest_dir, file)
_maybe_copy(src, dest_file, verbose)
else:
src = path
_maybe_copy(src, full_dest, verbose)
def _on_ipython_dir_changed(self, change):
ensure_dir_exists(self.ipython_extension_dir)
def install_nbextension(files, overwrite=False, symlink=False, ipython_dir=None, verbose=1):
"""Install a Javascript extension for the notebook
Stages files and/or directories into IPYTHONDIR/nbextensions.
By default, this compares modification time, and only stages files that need updating.
If `overwrite` is specified, matching files are purged before proceeding.
Parameters
----------
files : list(paths or URLs)
One or more paths or URLs to existing files directories to install.
These will be installed with their base name, so '/path/to/foo'
will install to 'nbextensions/foo'.
Archives (zip or tarballs) will be extracted into the nbextensions directory.
overwrite : bool [default: False]
If True, always install the files, regardless of what may already be installed.
symlink : bool [default: False]
If True, create a symlink in nbextensions, rather than copying files.
Not allowed with URLs or archives. Windows support for symlinks requires
Vista or above, Python 3, and a permission bit which only admin users
have by default, so don't rely on it.
ipython_dir : str [optional]
The path to an IPython directory, if the default value is not desired.
get_ipython_dir() is used by default.
verbose : int [default: 1]
Set verbosity level. The default is 1, where file actions are printed.
set verbose=2 for more output, or verbose=0 for silence.
"""
ipython_dir = ipython_dir or get_ipython_dir()
nbext = pjoin(ipython_dir, u'nbextensions')
# make sure nbextensions dir exists
ensure_dir_exists(nbext)
if isinstance(files, string_types):
# one file given, turn it into a list
files = [files]
for path in map(cast_unicode_py2, files):
if path.startswith(('https://', 'http://')):
if symlink:
raise ValueError("Cannot symlink from URLs")
# Given a URL, download it
with TemporaryDirectory() as td:
filename = urlparse(path).path.split('/')[-1]
local_path = os.path.join(td, filename)
if verbose >= 1:
print("downloading %s to %s" % (path, local_path))
urlretrieve(path, local_path)
# now install from the local copy
install_nbextension(
local_path, overwrite, symlink, ipython_dir, verbose)
continue
# handle archives
archive = None
if path.endswith('.zip'):
archive = zipfile.ZipFile(path)
elif _safe_is_tarfile(path):
archive = tarfile.open(path)
if archive:
if symlink:
raise ValueError("Cannot symlink from archives")
if verbose >= 1:
print("extracting %s to %s" % (path, nbext))
archive.extractall(nbext)
archive.close()
continue
dest = pjoin(nbext, basename(path))
if overwrite and os.path.exists(dest):
if verbose >= 1:
print("removing %s" % dest)
if os.path.isdir(dest) and not os.path.islink(dest):
shutil.rmtree(dest)
else:
os.remove(dest)
if symlink:
path = os.path.abspath(path)
if not os.path.exists(dest):
if verbose >= 1:
print("symlink %s -> %s" % (dest, path))
os.symlink(path, dest)
continue
if os.path.isdir(path):
strip_prefix_len = len(path) - len(basename(path))
for parent, dirs, files in os.walk(path):
dest_dir = pjoin(nbext, parent[strip_prefix_len:])
if not os.path.exists(dest_dir):
if verbose >= 2:
print("making directory %s" % dest_dir)
os.makedirs(dest_dir)
for file in files:
src = pjoin(parent, file)
# print("%r, %r" % (dest_dir, file))
dest = pjoin(dest_dir, file)
_maybe_copy(src, dest, verbose)
else:
src = path
_maybe_copy(src, dest, verbose)
def _on_ipython_dir_changed(self):
ensure_dir_exists(self.ipython_extension_dir)
import torch.nn.functional as F
import scipy
import random
from IPython.utils.path import ensure_dir_exists
import json
from numpy import genfromtxt
from torch.utils.data import Dataset, DataLoader
import matplotlib
matplotlib.use("agg")
import matplotlib.pyplot as plt
import shutil
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
cough_path = root_path + "Cough_Analytics/"
ensure_dir_exists(cough_path)
data_path = cough_path + "Data/"
ensure_dir_exists(data_path)
save_model_path = root_path + "Cough_Analytics/Models/"
ensure_dir_exists(save_model_path)
with open(data_path + "all_patient_keys.txt",'rb') as f:
data = f.readlines()
all_patient_keys = data[0]
print(all_patient_keys)
print("got all keys?",len(all_patient_keys))
def install_nbextension(files,
overwrite=False,
symlink=False,
user=False,
prefix=None,
nbextensions_dir=None,
verbose=1):
"""Install a Javascript extension for the notebook
Stages files and/or directories into the nbextensions directory.
By default, this compares modification time, and only stages files that need updating.
If `overwrite` is specified, matching files are purged before proceeding.
Parameters
----------
files : list(paths or URLs)
One or more paths or URLs to existing files directories to install.
These will be installed with their base name, so '/path/to/foo'
will install to 'nbextensions/foo'.
Archives (zip or tarballs) will be extracted into the nbextensions directory.
overwrite : bool [default: False]
If True, always install the files, regardless of what may already be installed.
symlink : bool [default: False]
If True, create a symlink in nbextensions, rather than copying files.
Not allowed with URLs or archives. Windows support for symlinks requires
Vista or above, Python 3, and a permission bit which only admin users
have by default, so don't rely on it.
user : bool [default: False]
Whether to install to the user's .ipython/nbextensions directory.
Otherwise do a system-wide install (e.g. /usr/local/share/jupyter/nbextensions).
prefix : str [optional]
Specify install prefix, if it should differ from default (e.g. /usr/local).
Will install to prefix/share/jupyter/nbextensions
nbextensions_dir : str [optional]
Specify absolute path of nbextensions directory explicitly.
verbose : int [default: 1]
Set verbosity level. The default is 1, where file actions are printed.
set verbose=2 for more output, or verbose=0 for silence.
"""
if sum(map(bool, [user, prefix, nbextensions_dir])) > 1:
raise ArgumentConflict(
"Cannot specify more than one of user, prefix, or nbextensions_dir."
)
if user:
nbext = pjoin(get_ipython_dir(), u'nbextensions')
else:
if prefix:
nbext = pjoin(prefix, 'share', 'jupyter', 'nbextensions')
elif nbextensions_dir:
nbext = nbextensions_dir
else:
nbext = SYSTEM_NBEXTENSIONS_INSTALL_DIR
# make sure nbextensions dir exists
ensure_dir_exists(nbext)
if isinstance(files, string_types):
# one file given, turn it into a list
files = [files]
for path in map(cast_unicode_py2, files):
if path.startswith(('https://', 'http://')):
if symlink:
raise ValueError("Cannot symlink from URLs")
# Given a URL, download it
with TemporaryDirectory() as td:
filename = urlparse(path).path.split('/')[-1]
local_path = os.path.join(td, filename)
if verbose >= 1:
print("downloading %s to %s" % (path, local_path))
urlretrieve(path, local_path)
# now install from the local copy
install_nbextension(local_path,
overwrite=overwrite,
symlink=symlink,
nbextensions_dir=nbext,
verbose=verbose)
continue
# handle archives
archive = None
if path.endswith('.zip'):
archive = zipfile.ZipFile(path)
elif _safe_is_tarfile(path):
archive = tarfile.open(path)
if archive:
if symlink:
raise ValueError("Cannot symlink from archives")
if verbose >= 1:
print("extracting %s to %s" % (path, nbext))
archive.extractall(nbext)
archive.close()
continue
dest = pjoin(nbext, basename(path))
if overwrite and os.path.exists(dest):
if verbose >= 1:
print("removing %s" % dest)
if os.path.isdir(dest) and not os.path.islink(dest):
shutil.rmtree(dest)
else:
os.remove(dest)
if symlink:
path = os.path.abspath(path)
if not os.path.exists(dest):
if verbose >= 1:
print("symlink %s -> %s" % (dest, path))
os.symlink(path, dest)
continue
if os.path.isdir(path):
strip_prefix_len = len(path) - len(basename(path))
for parent, dirs, files in os.walk(path):
dest_dir = pjoin(nbext, parent[strip_prefix_len:])
if not os.path.exists(dest_dir):
if verbose >= 2:
print("making directory %s" % dest_dir)
os.makedirs(dest_dir)
for file in files:
src = pjoin(parent, file)
# print("%r, %r" % (dest_dir, file))
dest = pjoin(dest_dir, file)
_maybe_copy(src, dest, verbose)
else:
src = path
_maybe_copy(src, dest, verbose)
#!/home/catskills/anaconda3/envs/xview2/bin/python
from IPython.utils.path import ensure_dir_exists
from pred_vars import *
for x in [VDIR, TESTDIR, INFER_DIR, POLYDIR, PREDICTION, COMBINED_JSON]:
ensure_dir_exists(x)
print(x)
from subprocess import call
from IPython.utils.path import ensure_dir_exists
# os.environ["CUDA_VISIBLE_DEVICES"]="1" # second gpu
VERSION = 'v_catskills_0.2.0'
PROJECT = 'xview2-catskills'
USERDIR = '/home/catskills/Desktop/'
CODEDIR = f'{USERDIR}{PROJECT}'
DATADIR = f'{USERDIR}dataxv2/'
TESTDIR = f'{DATADIR}test/images/'
SUBMIT_DIR = f'{DATADIR}{VERSION}_submit_001'
MODEL_DIR = f'/home/catskills/Desktop/dataxv2/release/{VERSION}/'
LOCALIZATION_MODEL = f'{MODEL_DIR}localization.hdf5'
DAMAGE_MODEL = f'{MODEL_DIR}classification.hdf5'
ensure_dir_exists(SUBMIT_DIR)
files = glob.glob(f'{TESTDIR}test_pre_*.png')
for pre_png in tqdm(files):
post_png = pre_png.replace('_pre_', '_post_')
image_id = pre_png.split('.')[0].split('/')[-1].split('_')[-1]
out_damage_png = f'{SUBMIT_DIR}/test_damage_{image_id}_prediction.png'
if os.path.isfile(out_damage_png):
continue
out_local_json = f'{SUBMIT_DIR}/test_localization_{image_id}_prediction.json'
out_local_png = f'{SUBMIT_DIR}/test_localization_{image_id}_prediction.png'
C = f'./inference.sh -x {CODEDIR} -i {pre_png} -p {post_png} -l {LOCALIZATION_MODEL} -c {DAMAGE_MODEL} -o {out_damage_png} -y'
call(C, shell=True)
if os.path.isfile(out_damage_png):
copyfile(out_damage_png, out_local_png)
def make_dirs_from_dict(d, current_dir):
for key, val in d.items():
ensure_dir_exists(os.path.join(current_dir, key))
if type(val) == dict:
make_dirs_from_dict(val, os.path.join(current_dir, key))
