def go_home(domain):
path_domain = Path(AQUATONE_ROOT, domain)
if path_domain.isdir():
path_domain.chdir()
else:
print("error %s is not a dir", domain)
sys.exit(1)
def load_inv_namespace(root_dir):
"""
Execute the :xfile:`tasks.py` file of this project and return its
`ns`.
"""
# self._tasks_loaded = True
tasks_file = root_dir.child('tasks.py')
if not tasks_file.exists():
return None
# raise Exception("No tasks.py file in {}".format(root_dir))
# return
# print("20180428 load tasks.py from {}".format(root_dir))
# http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path
# http://stackoverflow.com/questions/19009932/import-arbitrary-python-source-file-python-3-3
# fqname = 'atelier.prj_%s' % self.index
cwd = Path().resolve()
root_dir.chdir()
m = dict()
m["__file__"] = str(tasks_file)
with open(tasks_file) as f:
exec(f.read(), m)
cwd.chdir()
return m['ns']
def load_inv_namespace(root_dir):
"""
Execute the :xfile:`tasks.py` file of this project and return its
`ns`.
"""
# self._tasks_loaded = True
tasks_file = root_dir.child('tasks.py')
if not tasks_file.exists():
return None
# raise Exception("No tasks.py file in {}".format(root_dir))
# return
# print("20180428 load tasks.py from {}".format(root_dir))
# http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path
# http://stackoverflow.com/questions/19009932/import-arbitrary-python-source-file-python-3-3
# fqname = 'atelier.prj_%s' % self.index
cwd = Path().resolve()
root_dir.chdir()
m = dict()
m["__file__"] = str(tasks_file)
with open(tasks_file) as f:
exec(f.read(), m)
cwd.chdir()
return m['ns']
def test_pdf_to_png(self):
testdir = Path(r"C:\tmp\pdfprocessing\test")
testdir.chdir()
input_file = "testpdf.pdf"
output_file = Path(r"C:\tmp\pdfprocessing\test\test_gs_pdf_to_png.png")
gs = GhostScript()
gs.pdf_to_png(input_file,output_file)
self.assertTrue(output_file.exists(),"File")
class cwd(object):
def __init__(self, cwd):
self.prev_cwd = FSPath.cwd()
self.cwd = Path(cwd)
if not self.cwd.exists():
self.cwd.mkdir(parents=True)
def __enter__(self):
self.cwd.chdir()
return self.cwd
def __exit__(self, type_, value, traceback):
self.prev_cwd.chdir()
def get_setup_info(root_dir):
if not root_dir.child('setup.py').exists():
raise RuntimeError(
"You must call 'fab' from a project's root directory.")
# sys.path.insert(0, root_dir)
# setup_module = __import__('setup')
# print 20140610, root_dir
# del sys.path[0]
# return getattr(setup_module, 'SETUP_INFO', None)
g = dict()
g['__name__'] = 'not_main'
cwd = Path().resolve()
root_dir.chdir()
execfile(root_dir.child('setup.py'), g)
cwd.chdir()
return g.get('SETUP_INFO')
def down2save(code, ktype='d', start=None, end=None, path='.'):
cwd = Path(path)
cwd.chdir()
df = get_data(code, ktype, start, end)
if df.empty:
return df
df = prep(df)
if df.empty:
return df
df_demo = get_demo(df)
# print(df_demo)
start, end = extract_start_end(df_demo)
name_list = [code, start, end, ktype]
filename = fmt_filename(name_list)
filename_demo = fmt_filename_demo(name_list)
save(df, filename)
save(df_demo, filename_demo, header=True)
def get_setup_info(root_dir):
"""
Return `SETUP_INFO` defined in the :xfile:`setup.py` file of the
specified `root_dir`.
"""
setup_file = root_dir.child('setup.py')
if not setup_file.exists():
# print("20180118 no setup.py file in {}".format(root_dir.absolute()))
return {}
# raise RuntimeError(
# "You must call 'inv' from a project's root directory.")
# sys.path.insert(0, root_dir)
# setup_module = __import__('setup')
# print 20140610, root_dir
# del sys.path[0]
# return getattr(setup_module, 'SETUP_INFO', None)
g = dict()
g['__name__'] = 'not_main'
# g['__file__'] = setup_file
cwd = Path().resolve()
root_dir.chdir()
with open("setup.py") as f:
code = compile(f.read(), "setup.py", 'exec')
try:
exec(code, g)
except SystemExit:
cwd.chdir()
raise Exception(
"Oops, {} called sys.exit().\n"
"Atelier requires the setup() call to be in a "
"\"if __name__ == '__main__':\" condition.".format(
setup_file))
cwd.chdir()
info = g.get('SETUP_INFO')
if info is None:
raise Exception(
"Oops, {} doesn't define a name SETUP_INFO.".format(
setup_file))
return info
def get_setup_info(root_dir):
"""
Return `SETUP_INFO` defined in the :xfile:`setup.py` file of the
specified `root_dir`.
"""
setup_file = root_dir.child('setup.py')
if not setup_file.exists():
# print("20180118 no setup.py file in {}".format(root_dir.absolute()))
return {}
# raise RuntimeError(
# "You must call 'inv' from a project's root directory.")
# sys.path.insert(0, root_dir)
# setup_module = __import__('setup')
# print 20140610, root_dir
# del sys.path[0]
# return getattr(setup_module, 'SETUP_INFO', None)
g = dict()
g['__name__'] = 'not_main'
# g['__file__'] = setup_file
cwd = Path().resolve()
root_dir.chdir()
with open("setup.py") as f:
code = compile(f.read(), "setup.py", 'exec')
try:
exec(code, g)
except SystemExit:
cwd.chdir()
raise Exception(
"Oops, {} called sys.exit().\n"
"Atelier requires the setup() call to be in a "
"\"if __name__ == '__main__':\" condition.".format(
setup_file))
cwd.chdir()
info = g.get('SETUP_INFO')
if info is None:
raise Exception(
"Oops, {} doesn't define a name SETUP_INFO.".format(
setup_file))
return info
def load_tasks(self):
"""Load the :xfile:`tasks.py` of this project."""
if self._tasks_loaded:
return
self._tasks_loaded = True
self.name = self.nickname
if not self.root_dir.child('tasks.py').exists():
return
fqname = 'atelier.prj_%s' % self.index
cwd = Path().resolve()
self.root_dir.chdir()
# print("20160121 pseudo-importing file %s %s/tasks.py " % (
# self, self.root_dir))
(fp, pathname, desc) = imp.find_module('tasks', [self.root_dir])
m = imp.load_module(fqname, fp, pathname, desc)
cwd.chdir()
assert hasattr(m, 'ns')
main_package = m.ns.main_package
self.ns = m.ns
if main_package is None:
return
self.name = main_package
# self.name = name
# removed 20140116:
# self.dist = pkg_resources.get_distribution(name)
self.module = import_module(main_package)
self.SETUP_INFO = get_setup_info(self.root_dir)
self.srcref_url = getattr(self.module, 'srcref_url', None)
self.doc_trees = getattr(self.module, 'doc_trees', self.doc_trees)
self.intersphinx_urls = getattr(
self.module, 'intersphinx_urls', {})
class Installer(object):
install_path = None
postactivate = None
postdeactivate = None
project_name = None
project_dir = None
var_dict = {}
post_run_command_stack = []
def __init__(self, project_dir, project_name, envwrapper=False,
*args, **kwargs):
self.project_dir = Path(project_dir).absolute()
self.project_name = project_name
self.is_envwrapper = envwrapper
# make all attributes overridable so that external applications
# can make use of the pattern and reset the variable names
for k, v in six.iteritems(kwargs):
setattr(self, k, v)
self.install_path = Path(self.project_dir, project_name)
self.install_path.mkdir()
self.install_path.chdir()
self._environment_cache = False
self._template_dir_cache = False
self._template_cache = False
@property
def venv_folder(self):
"""
extracts the venv folder from the environment variables ($WORKON_HOME,
to be precise and combines it with the project name.
"""
path = os.environ.copy().get('WORKON_HOME')
if path:
return Path(path)
else:
return None
@property
def template_env(self):
"""
provides the template environment
"""
if not getattr(self, '_environment_cache', False):
self._environment_cache = Environment(
loader=FileSystemLoader(self.get_template_dir())
)
return self._environment_cache
def run_command(self, command, blocking=False):
command = Command(command)
command()
if blocking:
logger.debug('Waiting for command to finish...')
command.wait()
return True
def finish_queued_commands(self):
finish_queued_commands()
def get_installer_name(self):
return self.__class__.__name__.lower()
def get_template_dir(self):
if not getattr(self, '_template_cache', False):
self._template_dir_cache = Path(Path(__file__).parent, 'templates')
return self._template_dir_cache
def get_template(self, which_one):
"""
provides a wrapper around jinja2 get_template. Caches the result.
returns a cached template
"""
if not getattr(self, '_template_cache', False):
self._template_cache = dict()
if not self._template_cache.get(which_one, False):
template_file = '%s.%s.sh' % (self.get_installer_name(), which_one)
self._template_cache[which_one] = \
self.template_env.get_template(template_file)
return self._template_cache[which_one]
def run_prepare_configuration(self):
raise NotImplementedError('Must be implemented in subclass')
def render_config_for_file_template(self, which_one):
logger.info('preparing config variables for %s ...' % which_one)
template = self.get_template(which_one=which_one)
contents = template.render(**self.var_dict)
setattr(self, '%s' % which_one, contents)
logger.info('...done')
def create_file(self, which_one):
self.render_config_for_file_template(which_one=which_one)
logger.info('Creating config files in parent dir: %s'
% self.install_path)
#gets self.postdeactivate if which_one=postdeactivate
contents = getattr(self, which_one)
logger.info('%s: Writing contents to file ...' % which_one)
p = Path(self.install_path, which_one)
#write configuration and append it to the file
p.write_file(contents, 'a+')
logger.info('...done')
def move_to_venv(self, which_one):
"""
Moves the created config_files into the bin folder to be executed.
Does this by first pasting all the contents of the temporary file
into the new or existing target file and then deleting the temp file.
"""
target = Path(self.venv_folder, self.project_name, 'bin', which_one)
source = Path(self.install_path, which_one)
logger.info('target: %s, move_orig: %s' % (target, source))
if source.exists():
logger.info('Moving %s into place ...' % which_one)
content = source.read_file()
#make sure the directory exists
if not target.parent.exists():
target.parent.mkdir(parents=True)
target.write_file(content, 'w+')
source.remove()
logger.info('...done')
def run_create_configuration(self):
self.create_file(which_one='postactivate')
self.create_file(which_one='postdeactivate')
def run_post_create_configuration(self):
pass
def run(self):
self.run_prepare_configuration()
self.run_create_configuration()
self.run_post_create_configuration()
def __call__(self, *args, **kwargs):
self.run()
def down2save_update(code, ktype='d', start=None, end=None, path='.'):
"""
get k_chart date and save them locally, and update them.
get k_chart data using `tushare.get_k_data`
save data locally with tab separated files
could append data line by line if getting newer
Parameters
----------
code : str
ktype : format '%Y-%m-%d-%H-%M'
start, end : str or None
`datetime.datetime` format '%Y-%m-%d-%H-%M'
e.g. '2017-12-12-23-59'
path : str
where to search files locally
Returns
-------
"""
cwd = Path(path)
cwd.chdir()
filename_demo_local = get_local(code, ktype, path)
if not isinstance(filename_demo_local, str):
return down2save(code, ktype, start, end, path)
df_demo_local = read_data(filename_demo_local)
# print(df_demo_local)
start_local, end_local = extract_start_end(df_demo_local)
if is_up_to_date(end_local, ktype):
print('data up to date')
return
name_list_demo_local = split_filename(filename_demo_local)
filename_local = fmt_filename(name_list_demo_local[:-1])
df = get_data(code, ktype, start, end)
if df.empty:
return df
df = prep(df)
if df.empty:
return df
df_newer = df.loc[df.index > df_demo_local.index[-1]]
if df_newer.empty:
return df_newer
df_demo_new = get_demo_new(df_demo_local, df_newer)
# print(df_demo_new)
_, end_newer = extract_start_end(df_demo_new)
filename_list_new = [code, start_local, end_newer, ktype]
filename_new = fmt_filename(filename_list_new)
save(df_newer, filename_local, mode='a')
os.rename(filename_local, filename_new)
print('rename %s %s' % (filename_local, filename_new))
filename_demo_new = fmt_filename_demo(filename_list_new)
save(df_demo_new, filename_demo_new, header=True)
os.remove(filename_demo_local)
print('remove %s' % filename_demo_local)
from utilityclasses import GhostScript,PdfTk,Convert
import os
from unipath import Path
import unittest
import tempfile
from ocrmethods import convert_image
import shutil
from ocrmethods import get_text_from_image
testdir = Path(r"C:\tmp\pdfprocessing\tests")
testdir.chdir()
convert = Convert()
class TestPIL(unittest.TestCase):
def test_id_times_roman(self):
convert(r"C:\tmp\pdfprocessing\tests\test_pdf_page_tnr_noimage.pdf",
r"C:\tmp\pdfprocessing\tests\test_pdf_page_tnr_noimage.tif")
class TestGhostScript(unittest.TestCase):
def setUp(self):
self.gs = GhostScript()
self.testdir = testdir
self.tempdir = tempfile.TemporaryDirectory()
# print("creating tempdir: {}".format(os.path.abspath(self.tempdir.name)))
def test_pdf_to_png(self):
input_file = os.path.abspath("test_pdf_page_tnr_noimage.pdf")
output_filename = "test_gs_pdf_to_png.png"
class ProjectInstaller(Installer):
flavor = 'django_custom'
git_repo = 'https://github.com/Libermentix/project_skeletton_directory.git'
def __init__(self, project_dir, project_name,
db_sudo=False, db_sudo_user=None, *args, **kwargs):
super(ProjectInstaller, self).__init__(
project_dir, project_name, *args, **kwargs
)
self.var_dict = dict(
project_dir=add_trailing_slash(project_dir),
project_name=add_trailing_slash(project_name)
)
self._tmp_dir = None
if self.install_path.exists():
self.install_path.rmtree()
self.db_installer = DatabaseInstaller(
project_dir=project_dir, project_name=project_name,
sudo=db_sudo, sudo_user=db_sudo_user
)
self.django_installer = DjangoInstaller(
project_dir=project_dir, project_name=project_name
)
def run(self):
self.run_prepare_configuration()
self.run_create_configuration()
self.run_post_create_configuration()
def run_prepare_configuration(self):
self.get_git_repo()
self.install_skeletton()
self.install_requirements()
def run_post_create_configuration(self):
"""
run the the post_run_command_stack
"""
self.db_installer()
self.django_installer()
self.move_to_venv(which_one='postactivate')
self.move_to_venv(which_one='activate')
self.finish_queued_commands()
#run the post create configuration command for the children
for item in self.db_installer.post_run_command_stack:
# should be a callable or None
if item: item()
for item in self.django_installer.post_run_command_stack:
# should be a callable or None
if item:
logger.info('%s: Executing a django_installer_script ...' % item)
item()
@property
def requirements_file(self):
return Path(
self.install_path, 'requirements', 'base.txt'
).absolute()
@property
def repo_dir(self):
#get last
directory = self.git_repo.split('/')[-1:][0]
#remove .git
directory = directory.split('.')[0]
return Path(self._tmp_dir, directory)
def create_tmp_dir(self):
# TODO:
# Account for existing project paths, here it should ask to remove
# or abort.
self._tmp_dir = Path(self.install_path, 'tmp')
self._tmp_dir.mkdir()
self._tmp_dir.chdir()
def delete_tmp_dir(self):
self.project_dir.chdir()
self._tmp_dir.rmtree()
def get_git_repo(self):
self.create_tmp_dir()
logger.info('Cloning repository ...')
if self.repo_dir.exists():
logger.info('Repo dir exists removing it...')
self.repo_dir.rmtree()
git.Git().clone(self.git_repo)
logger.info('..done')
def install_skeletton(self):
logger.info('Installing %s' % self.flavor)
source = Path(self.repo_dir, self.flavor)
#move all items in the directory into the install_path
for item in source.listdir():
item.move(self.install_path)
self.delete_tmp_dir()
logger.info('...done')
def install_virtualenv(self):
"""
Calls a script that creates the virtual environment and installs
its dependencies, currently only sports python2.7 support.
"""
exec_path = Path(Path(__file__).parent, 'bash', 'installer.sh')
command = '%s %s %s %s' % (exec_path,
self.install_path,
self.project_name,
self.requirements_file)
logger.info('Installing virtualenv... (calling %s)' % command)
self.run_command(command)
def install_requirements(self):
if not self.is_envwrapper:
self.install_virtualenv()
else:
# we can assume that we are in the virtualenv now, and mkproject
# was called
command = 'pip install -r %s' % self.requirements_file
self.run_command(command)