def go_home(domain): path_domain = Path(AQUATONE_ROOT, domain) if path_domain.isdir(): path_domain.chdir() else: print("error %s is not a dir", domain) sys.exit(1)
def load_inv_namespace(root_dir): """ Execute the :xfile:`tasks.py` file of this project and return its `ns`. """ # self._tasks_loaded = True tasks_file = root_dir.child('tasks.py') if not tasks_file.exists(): return None # raise Exception("No tasks.py file in {}".format(root_dir)) # return # print("20180428 load tasks.py from {}".format(root_dir)) # http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path # http://stackoverflow.com/questions/19009932/import-arbitrary-python-source-file-python-3-3 # fqname = 'atelier.prj_%s' % self.index cwd = Path().resolve() root_dir.chdir() m = dict() m["__file__"] = str(tasks_file) with open(tasks_file) as f: exec(f.read(), m) cwd.chdir() return m['ns']
def load_inv_namespace(root_dir): """ Execute the :xfile:`tasks.py` file of this project and return its `ns`. """ # self._tasks_loaded = True tasks_file = root_dir.child('tasks.py') if not tasks_file.exists(): return None # raise Exception("No tasks.py file in {}".format(root_dir)) # return # print("20180428 load tasks.py from {}".format(root_dir)) # http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path # http://stackoverflow.com/questions/19009932/import-arbitrary-python-source-file-python-3-3 # fqname = 'atelier.prj_%s' % self.index cwd = Path().resolve() root_dir.chdir() m = dict() m["__file__"] = str(tasks_file) with open(tasks_file) as f: exec(f.read(), m) cwd.chdir() return m['ns']
def test_pdf_to_png(self): testdir = Path(r"C:\tmp\pdfprocessing\test") testdir.chdir() input_file = "testpdf.pdf" output_file = Path(r"C:\tmp\pdfprocessing\test\test_gs_pdf_to_png.png") gs = GhostScript() gs.pdf_to_png(input_file,output_file) self.assertTrue(output_file.exists(),"File")
class cwd(object): def __init__(self, cwd): self.prev_cwd = FSPath.cwd() self.cwd = Path(cwd) if not self.cwd.exists(): self.cwd.mkdir(parents=True) def __enter__(self): self.cwd.chdir() return self.cwd def __exit__(self, type_, value, traceback): self.prev_cwd.chdir()
def get_setup_info(root_dir): if not root_dir.child('setup.py').exists(): raise RuntimeError( "You must call 'fab' from a project's root directory.") # sys.path.insert(0, root_dir) # setup_module = __import__('setup') # print 20140610, root_dir # del sys.path[0] # return getattr(setup_module, 'SETUP_INFO', None) g = dict() g['__name__'] = 'not_main' cwd = Path().resolve() root_dir.chdir() execfile(root_dir.child('setup.py'), g) cwd.chdir() return g.get('SETUP_INFO')
def down2save(code, ktype='d', start=None, end=None, path='.'): cwd = Path(path) cwd.chdir() df = get_data(code, ktype, start, end) if df.empty: return df df = prep(df) if df.empty: return df df_demo = get_demo(df) # print(df_demo) start, end = extract_start_end(df_demo) name_list = [code, start, end, ktype] filename = fmt_filename(name_list) filename_demo = fmt_filename_demo(name_list) save(df, filename) save(df_demo, filename_demo, header=True)
def get_setup_info(root_dir): """ Return `SETUP_INFO` defined in the :xfile:`setup.py` file of the specified `root_dir`. """ setup_file = root_dir.child('setup.py') if not setup_file.exists(): # print("20180118 no setup.py file in {}".format(root_dir.absolute())) return {} # raise RuntimeError( # "You must call 'inv' from a project's root directory.") # sys.path.insert(0, root_dir) # setup_module = __import__('setup') # print 20140610, root_dir # del sys.path[0] # return getattr(setup_module, 'SETUP_INFO', None) g = dict() g['__name__'] = 'not_main' # g['__file__'] = setup_file cwd = Path().resolve() root_dir.chdir() with open("setup.py") as f: code = compile(f.read(), "setup.py", 'exec') try: exec(code, g) except SystemExit: cwd.chdir() raise Exception( "Oops, {} called sys.exit().\n" "Atelier requires the setup() call to be in a " "\"if __name__ == '__main__':\" condition.".format( setup_file)) cwd.chdir() info = g.get('SETUP_INFO') if info is None: raise Exception( "Oops, {} doesn't define a name SETUP_INFO.".format( setup_file)) return info
def get_setup_info(root_dir): """ Return `SETUP_INFO` defined in the :xfile:`setup.py` file of the specified `root_dir`. """ setup_file = root_dir.child('setup.py') if not setup_file.exists(): # print("20180118 no setup.py file in {}".format(root_dir.absolute())) return {} # raise RuntimeError( # "You must call 'inv' from a project's root directory.") # sys.path.insert(0, root_dir) # setup_module = __import__('setup') # print 20140610, root_dir # del sys.path[0] # return getattr(setup_module, 'SETUP_INFO', None) g = dict() g['__name__'] = 'not_main' # g['__file__'] = setup_file cwd = Path().resolve() root_dir.chdir() with open("setup.py") as f: code = compile(f.read(), "setup.py", 'exec') try: exec(code, g) except SystemExit: cwd.chdir() raise Exception( "Oops, {} called sys.exit().\n" "Atelier requires the setup() call to be in a " "\"if __name__ == '__main__':\" condition.".format( setup_file)) cwd.chdir() info = g.get('SETUP_INFO') if info is None: raise Exception( "Oops, {} doesn't define a name SETUP_INFO.".format( setup_file)) return info
def load_tasks(self): """Load the :xfile:`tasks.py` of this project.""" if self._tasks_loaded: return self._tasks_loaded = True self.name = self.nickname if not self.root_dir.child('tasks.py').exists(): return fqname = 'atelier.prj_%s' % self.index cwd = Path().resolve() self.root_dir.chdir() # print("20160121 pseudo-importing file %s %s/tasks.py " % ( # self, self.root_dir)) (fp, pathname, desc) = imp.find_module('tasks', [self.root_dir]) m = imp.load_module(fqname, fp, pathname, desc) cwd.chdir() assert hasattr(m, 'ns') main_package = m.ns.main_package self.ns = m.ns if main_package is None: return self.name = main_package # self.name = name # removed 20140116: # self.dist = pkg_resources.get_distribution(name) self.module = import_module(main_package) self.SETUP_INFO = get_setup_info(self.root_dir) self.srcref_url = getattr(self.module, 'srcref_url', None) self.doc_trees = getattr(self.module, 'doc_trees', self.doc_trees) self.intersphinx_urls = getattr( self.module, 'intersphinx_urls', {})
class Installer(object): install_path = None postactivate = None postdeactivate = None project_name = None project_dir = None var_dict = {} post_run_command_stack = [] def __init__(self, project_dir, project_name, envwrapper=False, *args, **kwargs): self.project_dir = Path(project_dir).absolute() self.project_name = project_name self.is_envwrapper = envwrapper # make all attributes overridable so that external applications # can make use of the pattern and reset the variable names for k, v in six.iteritems(kwargs): setattr(self, k, v) self.install_path = Path(self.project_dir, project_name) self.install_path.mkdir() self.install_path.chdir() self._environment_cache = False self._template_dir_cache = False self._template_cache = False @property def venv_folder(self): """ extracts the venv folder from the environment variables ($WORKON_HOME, to be precise and combines it with the project name. """ path = os.environ.copy().get('WORKON_HOME') if path: return Path(path) else: return None @property def template_env(self): """ provides the template environment """ if not getattr(self, '_environment_cache', False): self._environment_cache = Environment( loader=FileSystemLoader(self.get_template_dir()) ) return self._environment_cache def run_command(self, command, blocking=False): command = Command(command) command() if blocking: logger.debug('Waiting for command to finish...') command.wait() return True def finish_queued_commands(self): finish_queued_commands() def get_installer_name(self): return self.__class__.__name__.lower() def get_template_dir(self): if not getattr(self, '_template_cache', False): self._template_dir_cache = Path(Path(__file__).parent, 'templates') return self._template_dir_cache def get_template(self, which_one): """ provides a wrapper around jinja2 get_template. Caches the result. returns a cached template """ if not getattr(self, '_template_cache', False): self._template_cache = dict() if not self._template_cache.get(which_one, False): template_file = '%s.%s.sh' % (self.get_installer_name(), which_one) self._template_cache[which_one] = \ self.template_env.get_template(template_file) return self._template_cache[which_one] def run_prepare_configuration(self): raise NotImplementedError('Must be implemented in subclass') def render_config_for_file_template(self, which_one): logger.info('preparing config variables for %s ...' % which_one) template = self.get_template(which_one=which_one) contents = template.render(**self.var_dict) setattr(self, '%s' % which_one, contents) logger.info('...done') def create_file(self, which_one): self.render_config_for_file_template(which_one=which_one) logger.info('Creating config files in parent dir: %s' % self.install_path) #gets self.postdeactivate if which_one=postdeactivate contents = getattr(self, which_one) logger.info('%s: Writing contents to file ...' % which_one) p = Path(self.install_path, which_one) #write configuration and append it to the file p.write_file(contents, 'a+') logger.info('...done') def move_to_venv(self, which_one): """ Moves the created config_files into the bin folder to be executed. Does this by first pasting all the contents of the temporary file into the new or existing target file and then deleting the temp file. """ target = Path(self.venv_folder, self.project_name, 'bin', which_one) source = Path(self.install_path, which_one) logger.info('target: %s, move_orig: %s' % (target, source)) if source.exists(): logger.info('Moving %s into place ...' % which_one) content = source.read_file() #make sure the directory exists if not target.parent.exists(): target.parent.mkdir(parents=True) target.write_file(content, 'w+') source.remove() logger.info('...done') def run_create_configuration(self): self.create_file(which_one='postactivate') self.create_file(which_one='postdeactivate') def run_post_create_configuration(self): pass def run(self): self.run_prepare_configuration() self.run_create_configuration() self.run_post_create_configuration() def __call__(self, *args, **kwargs): self.run()
def down2save_update(code, ktype='d', start=None, end=None, path='.'): """ get k_chart date and save them locally, and update them. get k_chart data using `tushare.get_k_data` save data locally with tab separated files could append data line by line if getting newer Parameters ---------- code : str ktype : format '%Y-%m-%d-%H-%M' start, end : str or None `datetime.datetime` format '%Y-%m-%d-%H-%M' e.g. '2017-12-12-23-59' path : str where to search files locally Returns ------- """ cwd = Path(path) cwd.chdir() filename_demo_local = get_local(code, ktype, path) if not isinstance(filename_demo_local, str): return down2save(code, ktype, start, end, path) df_demo_local = read_data(filename_demo_local) # print(df_demo_local) start_local, end_local = extract_start_end(df_demo_local) if is_up_to_date(end_local, ktype): print('data up to date') return name_list_demo_local = split_filename(filename_demo_local) filename_local = fmt_filename(name_list_demo_local[:-1]) df = get_data(code, ktype, start, end) if df.empty: return df df = prep(df) if df.empty: return df df_newer = df.loc[df.index > df_demo_local.index[-1]] if df_newer.empty: return df_newer df_demo_new = get_demo_new(df_demo_local, df_newer) # print(df_demo_new) _, end_newer = extract_start_end(df_demo_new) filename_list_new = [code, start_local, end_newer, ktype] filename_new = fmt_filename(filename_list_new) save(df_newer, filename_local, mode='a') os.rename(filename_local, filename_new) print('rename %s %s' % (filename_local, filename_new)) filename_demo_new = fmt_filename_demo(filename_list_new) save(df_demo_new, filename_demo_new, header=True) os.remove(filename_demo_local) print('remove %s' % filename_demo_local)
from utilityclasses import GhostScript,PdfTk,Convert import os from unipath import Path import unittest import tempfile from ocrmethods import convert_image import shutil from ocrmethods import get_text_from_image testdir = Path(r"C:\tmp\pdfprocessing\tests") testdir.chdir() convert = Convert() class TestPIL(unittest.TestCase): def test_id_times_roman(self): convert(r"C:\tmp\pdfprocessing\tests\test_pdf_page_tnr_noimage.pdf", r"C:\tmp\pdfprocessing\tests\test_pdf_page_tnr_noimage.tif") class TestGhostScript(unittest.TestCase): def setUp(self): self.gs = GhostScript() self.testdir = testdir self.tempdir = tempfile.TemporaryDirectory() # print("creating tempdir: {}".format(os.path.abspath(self.tempdir.name))) def test_pdf_to_png(self): input_file = os.path.abspath("test_pdf_page_tnr_noimage.pdf") output_filename = "test_gs_pdf_to_png.png"
class ProjectInstaller(Installer): flavor = 'django_custom' git_repo = 'https://github.com/Libermentix/project_skeletton_directory.git' def __init__(self, project_dir, project_name, db_sudo=False, db_sudo_user=None, *args, **kwargs): super(ProjectInstaller, self).__init__( project_dir, project_name, *args, **kwargs ) self.var_dict = dict( project_dir=add_trailing_slash(project_dir), project_name=add_trailing_slash(project_name) ) self._tmp_dir = None if self.install_path.exists(): self.install_path.rmtree() self.db_installer = DatabaseInstaller( project_dir=project_dir, project_name=project_name, sudo=db_sudo, sudo_user=db_sudo_user ) self.django_installer = DjangoInstaller( project_dir=project_dir, project_name=project_name ) def run(self): self.run_prepare_configuration() self.run_create_configuration() self.run_post_create_configuration() def run_prepare_configuration(self): self.get_git_repo() self.install_skeletton() self.install_requirements() def run_post_create_configuration(self): """ run the the post_run_command_stack """ self.db_installer() self.django_installer() self.move_to_venv(which_one='postactivate') self.move_to_venv(which_one='activate') self.finish_queued_commands() #run the post create configuration command for the children for item in self.db_installer.post_run_command_stack: # should be a callable or None if item: item() for item in self.django_installer.post_run_command_stack: # should be a callable or None if item: logger.info('%s: Executing a django_installer_script ...' % item) item() @property def requirements_file(self): return Path( self.install_path, 'requirements', 'base.txt' ).absolute() @property def repo_dir(self): #get last directory = self.git_repo.split('/')[-1:][0] #remove .git directory = directory.split('.')[0] return Path(self._tmp_dir, directory) def create_tmp_dir(self): # TODO: # Account for existing project paths, here it should ask to remove # or abort. self._tmp_dir = Path(self.install_path, 'tmp') self._tmp_dir.mkdir() self._tmp_dir.chdir() def delete_tmp_dir(self): self.project_dir.chdir() self._tmp_dir.rmtree() def get_git_repo(self): self.create_tmp_dir() logger.info('Cloning repository ...') if self.repo_dir.exists(): logger.info('Repo dir exists removing it...') self.repo_dir.rmtree() git.Git().clone(self.git_repo) logger.info('..done') def install_skeletton(self): logger.info('Installing %s' % self.flavor) source = Path(self.repo_dir, self.flavor) #move all items in the directory into the install_path for item in source.listdir(): item.move(self.install_path) self.delete_tmp_dir() logger.info('...done') def install_virtualenv(self): """ Calls a script that creates the virtual environment and installs its dependencies, currently only sports python2.7 support. """ exec_path = Path(Path(__file__).parent, 'bash', 'installer.sh') command = '%s %s %s %s' % (exec_path, self.install_path, self.project_name, self.requirements_file) logger.info('Installing virtualenv... (calling %s)' % command) self.run_command(command) def install_requirements(self): if not self.is_envwrapper: self.install_virtualenv() else: # we can assume that we are in the virtualenv now, and mkproject # was called command = 'pip install -r %s' % self.requirements_file self.run_command(command)