def do_main(source, dest, password, encrypt, # pylint: disable=W0621
crypt_file, crypt_map, regex=None, fLOG=None):
"""
Encrypts or decrypts a folder, see @see cl EncryptedBackup.
The function relies on module :epkg:`pycrypto`, :epkg:`cryptography`,
algoritm `AES <https://fr.wikipedia.org/wiki/Advanced_Encryption_Standard>`_,
`Fernet <https://cryptography.io/en/latest/fernet/>`_.
@param source source of files to encrypt or decrypt
@param dest destination
@param password password
@param encrypt boolean, True to encrypt
@param crypt_file encrypted file
@param crypt_map @see cl EncryptedBackup
@param regex regular expression to filter in files to retrieve
@param fLOG logging function
"""
if not os.path.exists(source):
raise FileNotFoundError(source) # pragma: no cover
try:
from pyquickhelper.filehelper import EncryptedBackup, TransferAPIFile, FileTreeNode
except ImportError: # pragma: no cover
folder = os.path.normpath(os.path.join(
os.path.abspath(os.path.dirname(__file__)), "..", ".."))
sys.path.append(folder)
from pyquickhelper.filehelper import EncryptedBackup, TransferAPIFile, FileTreeNode
if isinstance(password, str):
password = bytes(password, encoding="ascii")
root = source
local = root
api = TransferAPIFile(dest)
if encrypt:
if fLOG:
fLOG("looking for file in", root)
ft = FileTreeNode(root, repository=False, fLOG=fLOG, log1=True)
enc = EncryptedBackup(key=password, file_tree_node=ft,
transfer_api=api, root_local=local, file_status=crypt_file,
file_map=crypt_map, fLOG=fLOG)
if fLOG:
fLOG("start backup")
issue = enc.start_transfering()[1]
for file, exc in issue:
if fLOG: # pragma: no cover
fLOG("{0} -- {1}".format(file, exc))
else:
enc = EncryptedBackup(key=password, file_tree_node=None,
transfer_api=api, root_local=None, file_status=None,
file_map=None, fLOG=fLOG)
if fLOG:
fLOG("start restoration")
enc.retrieve_all(source, regex=regex)
def test_transfer_ftp_start_transfering(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
ftp = self.get_web()
if ftp is None:
return
# Transfering
now = datetime.datetime.now()
temp = get_temp_folder(__file__, "temp_transfer_ftp_true")
with open(os.path.join(temp, "essai.txt"), 'w') as f:
f.write(str(now))
sfile = os.path.join(temp, "status_ut.txt")
ftn = FileTreeNode(temp)
web = FolderTransferFTP(ftn,
ftp,
sfile,
root_web="/home/ftpuser/ftp/web/apptest",
fLOG=fLOG,
exc=True)
done = web.start_transfering(delay=0.1)
ftp.close()
names = [os.path.split(f.filename)[-1] for f in done]
self.assertEqual(names, ['essai.txt'])
# two, same file, should not be done again
ftp = self.get_web()
if ftp is None:
return
web = FolderTransferFTP(ftn,
ftp,
sfile,
root_web="/home/ftpuser/ftp/web/apptest/",
fLOG=fLOG)
done = web.start_transfering(delay=0.1)
ftp.close()
self.assertEmpty(done)
ftp = self.get_web()
if ftp is None:
return
content = ftp.retrieve("/home/ftpuser/ftp/web/apptest/", "essai.txt")
self.assertNotEmpty(content)
ftp.close()
def test_backup_file(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
try:
import Crypto as skip__
algo = "AES"
except ImportError:
algo = "fernet"
temp = get_temp_folder(__file__, "temp_backup_files_file")
root = os.path.normpath(os.path.join(temp, ".."))
fLOG(root)
api = TransferAPIFile(os.path.join(temp, "backup"))
ft = FileTreeNode(root, filter=".*[.]py", repository=False)
enc = EncryptedBackup(key=b"unit" * 8,
file_tree_node=ft,
transfer_api=api,
file_status=os.path.join(temp, "status.txt"),
file_map=os.path.join(temp, "mapping.txt"),
root_local=os.path.join(temp, "..", ".."),
threshold_size=2000,
fLOG=fLOG,
algo=algo)
done, issue = enc.start_transfering()
self.assertNotEmpty(done)
self.assertEmpty(issue)
for k, v in sorted(enc.Mapping.items()):
fLOG(k, len(v.pieces), v)
enc.load_mapping()
outfile = os.path.join(temp, "backed_test_backup_file.py")
fpth = "ut_filehelper\\test_backup_file.py"
if not sys.platform.startswith("win"):
fpth = fpth.replace("\\", "/")
s = enc.retrieve(fpth, filename=outfile)
with open(outfile, "r") as f:
c2 = f.read()
with open(__file__.replace(".pyc", ".py"), "r") as f:
c1 = f.read()
self.assertEqual(c1, c2)
detemp = os.path.join(temp, "retrieved")
s = enc.retrieve_all(detemp, regex=".*[.]py")
self.assertNotEmpty(s)
def test_file_tree_node(self):
this = os.path.abspath(os.path.dirname(__file__))
folder = os.path.normpath(os.path.join(this, "..", "..", "src"))
def filter(root, path, f, d):
return "__pycache__" not in path and "__pycache__" not in f
ftn = FileTreeNode(folder, log=True, log1=True,
filter=filter)
if len(ftn) == 2:
raise Exception("%d" % len(ftn))
nb = 0
nrst = 0
for f in ftn:
if f.isfile():
hash = f.hash_md5_readfile()
s = str(f)
self.assertNotEmpty(s)
self.assertNotEmpty(hash)
nb += 1
if nb > 15:
break
if "cli_helper.py" in f.name:
continue
if "__init__" not in f.name and ".py" in f.name and ".pyc" not in f.name \
and "__main__" not in f.name:
content = f.get_content()
rst = filecontent_to_rst(f.fullname, content)
contr, doc = rst
nrst += 1
self.assertNotIn("no documentation", doc)
self.assertIn(".. _f-", contr)
try:
cont2 = replace_relative_import_fct(f.fullname)
except ValueError as e:
raise ValueError(
"Unable to process '{}'.".format(f.fullname)) from e
lines = cont2.split("\n")
condition = "# replace # from ." in cont2
if not condition:
for line in lines:
if "from ." in line and "import" in line:
doc = "\n-------------DOC--------\n" + doc
raise Exception(
"{0}\nLINE:\n{1}\n-------CONT---------:\n{2}{3}".format(
f.fullname, line, cont2, doc))
self.assertGreater(nb, 0)
self.assertGreater(nrst, 0)
def test_folder_transfer(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
temp = get_temp_folder(__file__, "temp_folder_transfer")
status = os.path.join(temp, "temp_status_file.txt")
folder = os.path.normpath(os.path.join(temp, "..", "..", "..", "src"))
ftn = FileTreeNode(folder)
ftp = MockTransferFTP("http://www.xavierdupre.fr/",
"login",
"password",
fLOG=fLOG)
fftp = FolderTransferFTP(
ftn,
ftp,
status,
footer_html="<b>footer</b>",
content_filter=lambda c, f, force_allow=False: c)
li = list(fftp.iter_eligible_files())
assert len(li) > 0
done = fftp.start_transfering()
assert len(done) > 0
fLOG(done[0])
fLOG(done)
assert os.path.exists(status)
done = fftp.start_transfering()
if len(done) > 0:
for f in done:
fLOG(f)
assert False
ftp.close()
def publish_documentation(docs, ftpsite=None, login=None, password=None,
footer_html=None, content_filter=trigger_on_specific_strings,
is_binary=content_as_binary, force_allow=None,
delay=0.5, exc=False, ftps='FTP',
page_transform=None, fLOG=print):
"""
Publishes the documentation and the setups of a python module on a webiste,
it assumes the modules is organized the same way as :epkg:`pyquickhelper`.
@param docs list of dictionaries (see below)
@param ftpsite something like ``ftp.something.``
@param login login
@param password password
@param footer_html append this HTML code to any uploaded page (such a javascript code to count the audience)
@param content_filter filter the content of a file (it raises an exception if the result is None),
appies only on text files
@param is_binary a function to tell if a content of a file is binary or not
@param force_allow a file is not published if it contains something which looks like credentials
except if this string is part of *force_allow*
@param delay delay between file transferring (in average)
@param exc raise exception if not able to transfer
@param ftps use protocol FTP, TLS, or SFTP
@param page_transform function which transforms
the page before uploading it,
@see fn text_transform
@param fLOG logging function
*docs* is a list of dictionaries which must contain for each folder
to transfer:
- ``local``: local folder
- ``root_local``: local paths will be related to this root
- ``root_web``: prefix to add to the remote paths
- ``status_file``: a file where the function populates the transfered files and some information about them
A local file is composed by ``<local_root>/<relative_path>``, it
will be uploaded to ``<web_root>/<relative_path>``.
"""
params = {"ftpsite": ftpsite,
"login": login,
"password": password}
nbnone = len([v for k, v in params.items() if v is None or len(v) == 0])
if nbnone > 0:
raise ValueError(
"One of the following parameters is not specified:\n{0}".format(params))
nbnone = [v for k, v in params.items() if v is None or len(v) == 0]
if len(nbnone) > 0:
raise Exception("one of the parameters is None:\n" + str(nbnone))
password = params["password"]
login = params["login"]
ftpsite = params["ftpsite"]
filter_out = "([/\\\\]((moduletoc.html)|(blogtoc.html)|(searchbox.html)))|([.]buildinfo)"
ftp = TransferFTP(ftpsite, login, password, ftps=ftps, fLOG=fLOG)
if page_transform is None:
fct_transform = text_transform
else:
def combined_transform(ftpp, filename, content):
text_transform(ftpp, filename, content)
page_transform(ftpp, filename, content)
fct_transform = combined_transform
for project in docs:
fLOG("######################################################################")
for k, v in sorted(project.items()):
fLOG("[publish_documentation] loop {}='{}'".format(k, v))
location = project["local"]
root_local = project["root_local"]
root_web = project["root_web"]
sfile = project["status_file"]
rootw = project["root_web"]
# documentation + setup
fLOG("[publish_documentation] location='{}'".format(location))
ftn = FileTreeNode(root_local)
fftp = FolderTransferFTP(ftn, ftp, sfile, root_web=rootw, fLOG=fLOG, footer_html=footer_html,
content_filter=content_filter, is_binary=is_binary,
text_transform=fct_transform, filter_out=filter_out,
force_allow=force_allow, exc=exc)
fftp.start_transfering(delay=delay)
ftn = FileTreeNode(os.path.join(root_local, ".."),
filter=lambda root, path, f, dir: not dir)
fftp = FolderTransferFTP(ftn, ftp, sfile,
root_web=root_web.replace("helpsphinx", ""), fLOG=fLOG,
footer_html=footer_html, content_filter=content_filter,
is_binary=is_binary, text_transform=fct_transform)
fftp.start_transfering()
ftp.close()
def publish_documentation(docs,
ftpsite=None,
login=None,
password=None,
key_save="my_password",
footer_html=None,
content_filter=trigger_on_specific_strings,
is_binary=content_as_binary,
force_allow=None,
fLOG=print):
"""
Publishes the documentation and the setups of a python module on a webiste,
it assumes the modules is organized the same way as
`pyquickhelper <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/index.html>`_.
@param docs list of dictionaries (see below)
@param ftpsite something like ``ftp.something.``
@param login login
@param password password
@param key_save see function `open_window_params <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/tkinterquickhelper/funcwin/frame_params.html#tkinterquickhelper.funcwin.frame_params.open_window_params>`_
@param footer_html append this HTML code to any uploaded page (such a javascript code to count the audience)
@param content_filter filter the content of a file (it raises an exception if the result is None),
appies only on text files
@param is_binary a function to tell if a content of a file is binary or not
@param force_allow a file is not published if it contains something which looks like credentials
except if this string is part of *force_allow*
@param fLOG logging function
*docs* is a list of dictionaries which must contain for each folder
to transfer:
- ``local``: local folder
- ``root_local``: local paths will be related to this root
- ``root_web``: prefix to add to the remote paths
- ``status_file``: a file where the function populates the transfered files and some information about them
A local file is composed by ``<local_root>/<relative_path>``, it
will be uploaded to ``<web_root>/<relative_path>``.
If one of the three first parameters is None, the function
will open a popup windows to ask the missing information.
See `open_window_params <http://www.xavierdupre.fr/app/tkinterquickhelper/helpsphinx/tkinterquickhelper/funcwin/frame_params.html#tkinterquickhelper.funcwin.frame_params.open_window_params>`_.
"""
params = {"ftpsite": ftpsite, "login": login, "password": password}
nbnone = len([v for k, v in params.items() if v is None or len(v) == 0])
if nbnone > 0:
from tkinterquickhelper.funcwin import open_window_params
fLOG("retrying to get parameters from users")
for k, v in sorted(params.items()):
fLOG(" {0}={1}".format(k, v))
params = open_window_params(params,
title="Website and Credentials",
help_string="ftp site + login + password",
key_save=key_save)
nbnone = [v for k, v in params.items() if v is None or len(v) == 0]
if len(nbnone) > 0:
raise Exception("one of the parameters is None:\n" + str(nbnone))
password = params["password"]
login = params["login"]
ftpsite = params["ftpsite"]
filter_out = "([/\\\\]((moduletoc.html)|(blogtoc.html)|(searchbox.html)))|([.]buildinfo)"
ftp = TransferFTP(ftpsite, login, password, fLOG=fLOG)
for project in docs:
fLOG(
"######################################################################"
)
for k, v in sorted(project.items()):
fLOG(" {}={}".format(k, v))
location = project["local"]
root_local = project["root_local"]
root_web = project["root_web"]
sfile = project["status_file"]
rootw = project["root_web"]
# documentation + setup
fLOG("-------------------------", location)
ftn = FileTreeNode(root_local)
fftp = FolderTransferFTP(ftn,
ftp,
sfile,
root_web=rootw,
fLOG=fLOG,
footer_html=footer_html,
content_filter=content_filter,
is_binary=is_binary,
text_transform=text_transform,
filter_out=filter_out,
force_allow=force_allow)
fftp.start_transfering()
ftn = FileTreeNode(os.path.join(root_local, ".."),
filter=lambda root, path, f, dir: not dir)
fftp = FolderTransferFTP(ftn,
ftp,
sfile,
root_web=root_web.replace("helpsphinx", ""),
fLOG=fLOG,
footer_html=footer_html,
content_filter=content_filter,
is_binary=is_binary,
text_transform=text_transform)
fftp.start_transfering()
ftp.close()
def test_transfer_ftp_start_transfering(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import keyring
prefix = "pyquickhelper,"
try:
user = keyring.get_password("web", prefix + "user")
pwd = keyring.get_password("web", prefix + "pwd")
ftpsite = keyring.get_password("web", prefix + "ftp")
except RuntimeError:
user = None
pwd = None
if user is None:
if not is_travis_or_appveyor():
raise Exception("user password is empty, prefix='{0}', username='******'".format(
prefix, get_user()))
return
# Transfering
now = datetime.datetime.now()
temp = get_temp_folder(__file__, "temp_transfer_ftp_true")
with open(os.path.join(temp, "essai.txt"), 'w') as f:
f.write(str(now))
sfile = os.path.join(temp, "status_ut.txt")
ftn = FileTreeNode(temp)
# one
try:
ftp = TransferFTP(ftpsite, user, pwd, fLOG=fLOG)
except ftplib.error_temp as e:
if "421 Home directory not available" in str(e):
return
raise e
except socket.gaierror as ee:
if "Name or service not known" in str(ee):
return
if "getaddrinfo failed" in str(ee):
return
raise ee
web = FolderTransferFTP(ftn, ftp, sfile,
root_web="/www/htdocs/apptest/",
fLOG=fLOG)
done = web.start_transfering(delay=0.1)
ftp.close()
names = [os.path.split(f.filename)[-1] for f in done]
self.assertEqual(names, ['essai.txt'])
# two, same file, should not be done again
ftp = TransferFTP(ftpsite, user, pwd, fLOG=fLOG)
web = FolderTransferFTP(ftn, ftp, sfile,
root_web="/www/htdocs/apptest/",
fLOG=fLOG)
done = web.start_transfering(delay=0.1)
ftp.close()
self.assertEmpty(done)