def test_transfer_ftp_true(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import keyring
prefix = "pyquickhelper,"
try:
user = keyring.get_password("web", prefix + "user")
pwd = keyring.get_password("web", prefix + "pwd")
except RuntimeError:
user = None
pwd = None
if user is None:
if not is_travis_or_appveyor():
zoo = []
for k, v in sorted(os.environ.items()):
zoo.append("{0}={1}".format(k, v))
raise Exception("user password is empty, prefix='{0}', username='******'\n{2}".format(
prefix, get_user(), "\n".join(zoo)))
return
web = TransferFTP("ftp.xavierdupre.fr", user, pwd, fLOG=fLOG)
r = web.ls(".")
fLOG(r)
self.assertTrue(isinstance(r, list))
web.close()
def ftp_list_modules(
ftp_location="/www/htdocs/enseignement/setup",
http_location="http://www.xavierdupre.fr/enseignement/setup",
filename="index_modules_list.html"):
"""
Update the list of backuped modules assuming they are stored on a FTP website.
It gets the list of wheels in a folder and creates a HTML pages.
It then uploads the final pages
@param ftp_location location on the website
@param http_location same location but on http protocol
@param filename name of the file to produce
@return list of modules
The module uses *keyring* to retrieve the credentials.
You can set them up with:
::
keyring.get_password("ftp_list_modules", os.environ["COMPUTERNAME"] + "site", "...")
keyring.get_password("ftp_list_modules", os.environ["COMPUTERNAME"] + "login", "...")
keyring.get_password("ftp_list_modules", os.environ["COMPUTERNAME"] + "password", "...")
"""
hostname = os.environ.get("COMPUTERNAME", os.environ.get("HOSTNAME", ""))
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import keyring
ftp_site = keyring.get_password("ftp_list_modules", hostname + "site")
login = keyring.get_password("ftp_list_modules", hostname + "login")
password = keyring.get_password("ftp_list_modules", hostname + "password")
if not ftp_site:
raise ValueError("ftp_site is empty, some missing keyring?")
if not login:
raise ValueError("login is empty")
if not password:
raise ValueError("password is empty")
ftp = TransferFTP(ftp_site, login, password)
res = ftp.ls(ftp_location)
rows = ["<html><body><h1>storage for unit test</h1>\n<ul>"]
ret = []
for i, v in enumerate(sorted(_["name"] for _ in res)):
if v in ('.', '..'):
continue
ret.append(v)
line = '<li>{1} - <a href="{2}/{0}">{0}</a></li>'.format(
v, i, http_location)
rows.append(line)
rows.append("</ul></body></html>")
content = "\n".join(rows)
bstr = content.encode('ascii')
ftp.transfer(bstr, ftp_location + "/", filename)
ftp.close()
return ret
def ftp_list_modules(ftp_location="/home/ftpuser/ftp/web/enseignement/setup",
http_location="http://www.xavierdupre.fr/enseignement/setup",
filename="index_modules_list.html",
ftps='SFTP'):
"""
Updates the list of backuped modules assuming they are stored on a FTP website.
It gets the list of wheels in a folder and creates a HTML pages.
It then uploads the final pages.
@param ftp_location location on the website
@param http_location same location but on http protocol
@param filename name of the file to produce
@return list of modules
The module uses *keyring* to retrieve the credentials.
You can set them up with:
::
from pyquickhelper.loghelper import get_password
get_password("ftp_list_modules", "ensae_teaching_cs2,site", "...")
get_password("ftp_list_modules", "ensae_teaching_cs2,login", "...")
get_password("ftp_list_modules", "ensae_teaching_cs2,password", "...")
"""
ftp_site = get_password(
"ftp_list_modules", "ensae_teaching_cs2,site")
login = get_password(
"ftp_list_modules", "ensae_teaching_cs2,login")
password = get_password(
"ftp_list_modules", "ensae_teaching_cs2,password")
if not ftp_site:
raise ValueError("ftp_site is empty, some missing keyring?")
if not login:
raise ValueError("login is empty")
if not password:
raise ValueError("password is empty")
ftp = TransferFTP(ftp_site, login, password, ftps=ftps)
res = ftp.ls(ftp_location)
rows = ["<html><body><h1>storage for unit test</h1>\n<ul>"]
ret = []
for i, v in enumerate(sorted(_["name"] for _ in res)):
if v in ('.', '..'):
continue
ret.append(v)
line = '<li>{1} - <a href="{2}/{0}">{0}</a></li>'.format(
v, i, http_location)
rows.append(line)
rows.append("</ul></body></html>")
content = "\n".join(rows)
bstr = content.encode('ascii')
ftp.transfer(bstr, ftp_location + "/", filename)
ftp.close()
return ret
def test_transfer_ftp_true(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
prefix = "pyquickhelper,"
try:
user = get_password("web", prefix + "user", ask=False)
pwd = get_password("web", prefix + "pwd", ask=False)
ftpsite = get_password("web", prefix + "ftp", ask=False)
except (RuntimeError, AttributeError):
user = None
pwd = None
if user is None:
if not is_travis_or_appveyor():
zoo = []
for k, v in sorted(os.environ.items()):
zoo.append("{0}={1}".format(k, v))
raise Exception(
"user password is empty, prefix='{0}', username='******'\n{2}"
.format(prefix, get_user(), "\n".join(zoo)))
return
try:
web = TransferFTP(ftpsite, user, pwd, fLOG=fLOG)
except ftplib.error_perm as e:
if "Non-anonymous sessions must use encryption." in str(e):
return
raise e
except ftplib.error_temp as e:
if "421 Home directory not available" in str(e):
return
raise e
except socket.gaierror as ee:
if "Name or service not known" in str(ee):
return
if "getaddrinfo failed" in str(ee):
return
raise ee
r = web.ls(".")
fLOG(r)
self.assertTrue(isinstance(r, list))
web.close()
def test_transfer_ftp_true(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import keyring
prefix = "pyquickhelper,"
try:
user = keyring.get_password("web", prefix + "user")
pwd = keyring.get_password("web", prefix + "pwd")
ftpsite = keyring.get_password("web", prefix + "ftp")
except RuntimeError:
user = None
pwd = None
if user is None:
if not is_travis_or_appveyor():
zoo = []
for k, v in sorted(os.environ.items()):
zoo.append("{0}={1}".format(k, v))
raise Exception("user password is empty, prefix='{0}', username='******'\n{2}".format(
prefix, get_user(), "\n".join(zoo)))
return
try:
web = TransferFTP(ftpsite, user, pwd, fLOG=fLOG)
except ftplib.error_temp as e:
if "421 Home directory not available" in str(e):
return
raise e
except socket.gaierror as ee:
if "Name or service not known" in str(ee):
return
if "getaddrinfo failed" in str(ee):
return
raise ee
r = web.ls(".")
fLOG(r)
self.assertTrue(isinstance(r, list))
web.close()
def ftp_list_modules(ftp_location="/www/htdocs/enseignement/setup",
http_location="http://www.xavierdupre.fr/enseignement/setup",
filename="index_modules_list.html"):
"""
Updates the list of backuped modules assuming they are stored on a FTP website.
It gets the list of wheels in a folder and creates a HTML pages.
It then uploads the final pages.
@param ftp_location location on the website
@param http_location same location but on http protocol
@param filename name of the file to produce
@return list of modules
The module uses *keyring* to retrieve the credentials.
You can set them up with:
::
keyring.get_password("ftp_list_modules", "ensae_teaching_cs,site", "...")
keyring.get_password("ftp_list_modules", "ensae_teaching_cs,login", "...")
keyring.get_password("ftp_list_modules", "ensae_teaching_cs,password", "...")
"""
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import keyring
ftp_site = keyring.get_password(
"ftp_list_modules", "ensae_teaching_cs,site")
login = keyring.get_password("ftp_list_modules", "ensae_teaching_cs,login")
password = keyring.get_password(
"ftp_list_modules", "ensae_teaching_cs,password")
if not ftp_site:
raise ValueError("ftp_site is empty, some missing keyring?")
if not login:
raise ValueError("login is empty")
if not password:
raise ValueError("password is empty")
ftp = TransferFTP(ftp_site, login, password)
res = ftp.ls(ftp_location)
rows = ["<html><body><h1>storage for unit test</h1>\n<ul>"]
ret = []
for i, v in enumerate(sorted(_["name"] for _ in res)):
if v in ('.', '..'):
continue
ret.append(v)
line = '<li>{1} - <a href="{2}/{0}">{0}</a></li>'.format(
v, i, http_location)
rows.append(line)
rows.append("</ul></body></html>")
content = "\n".join(rows)
bstr = content.encode('ascii')
ftp.transfer(bstr, ftp_location + "/", filename)
ftp.close()
return ret
def test_transfer_ftp_start_transfering(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import keyring
prefix = "pyquickhelper,"
try:
user = keyring.get_password("web", prefix + "user")
pwd = keyring.get_password("web", prefix + "pwd")
except RuntimeError:
user = None
pwd = None
if user is None:
if not is_travis_or_appveyor():
raise Exception("user password is empty, prefix='{0}', username='******'".format(
prefix, get_user()))
return
# Transfering
now = datetime.datetime.now()
temp = get_temp_folder(__file__, "temp_transfer_ftp_true")
with open(os.path.join(temp, "essai.txt"), 'w') as f:
f.write(str(now))
sfile = os.path.join(temp, "status_ut.txt")
ftn = FileTreeNode(temp)
# one
ftp = TransferFTP("ftp.xavierdupre.fr", user, pwd, fLOG=fLOG)
web = FolderTransferFTP(ftn, ftp, sfile,
root_web="/www/htdocs/apptest/",
fLOG=fLOG)
done = web.start_transfering(delay=0.1)
ftp.close()
names = [os.path.split(f.filename)[-1] for f in done]
self.assertEqual(names, ['essai.txt'])
# two, same file, should not be done again
ftp = TransferFTP("ftp.xavierdupre.fr", user, pwd, fLOG=fLOG)
web = FolderTransferFTP(ftn, ftp, sfile,
root_web="/www/htdocs/apptest/",
fLOG=fLOG)
done = web.start_transfering(delay=0.1)
ftp.close()
self.assertEmpty(done)
def get_web(self):
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import keyring
prefix = "pyquickhelper2,"
try:
user = keyring.get_password("web", prefix + "user")
pwd = keyring.get_password("web", prefix + "pwd")
ftpsite = keyring.get_password("web", prefix + "ftp")
except RuntimeError:
user = None
pwd = None
if user is None:
if not is_travis_or_appveyor():
zoo = []
for k, v in sorted(os.environ.items()):
zoo.append("{0}={1}".format(k, v))
raise Exception(
"user password is empty, prefix='{0}', username='******'\n{2}"
.format(prefix, get_user(), "\n".join(zoo)))
return None
web = TransferFTP(ftpsite, user, pwd, fLOG=fLOG, ftps='SFTP')
return web
def publish_documentation(docs, ftpsite=None, login=None, password=None,
footer_html=None, content_filter=trigger_on_specific_strings,
is_binary=content_as_binary, force_allow=None,
delay=0.5, exc=False, ftps='FTP',
page_transform=None, fLOG=print):
"""
Publishes the documentation and the setups of a python module on a webiste,
it assumes the modules is organized the same way as :epkg:`pyquickhelper`.
@param docs list of dictionaries (see below)
@param ftpsite something like ``ftp.something.``
@param login login
@param password password
@param footer_html append this HTML code to any uploaded page (such a javascript code to count the audience)
@param content_filter filter the content of a file (it raises an exception if the result is None),
appies only on text files
@param is_binary a function to tell if a content of a file is binary or not
@param force_allow a file is not published if it contains something which looks like credentials
except if this string is part of *force_allow*
@param delay delay between file transferring (in average)
@param exc raise exception if not able to transfer
@param ftps use protocol FTP, TLS, or SFTP
@param page_transform function which transforms
the page before uploading it,
@see fn text_transform
@param fLOG logging function
*docs* is a list of dictionaries which must contain for each folder
to transfer:
- ``local``: local folder
- ``root_local``: local paths will be related to this root
- ``root_web``: prefix to add to the remote paths
- ``status_file``: a file where the function populates the transfered files and some information about them
A local file is composed by ``<local_root>/<relative_path>``, it
will be uploaded to ``<web_root>/<relative_path>``.
"""
params = {"ftpsite": ftpsite,
"login": login,
"password": password}
nbnone = len([v for k, v in params.items() if v is None or len(v) == 0])
if nbnone > 0:
raise ValueError(
"One of the following parameters is not specified:\n{0}".format(params))
nbnone = [v for k, v in params.items() if v is None or len(v) == 0]
if len(nbnone) > 0:
raise Exception("one of the parameters is None:\n" + str(nbnone))
password = params["password"]
login = params["login"]
ftpsite = params["ftpsite"]
filter_out = "([/\\\\]((moduletoc.html)|(blogtoc.html)|(searchbox.html)))|([.]buildinfo)"
ftp = TransferFTP(ftpsite, login, password, ftps=ftps, fLOG=fLOG)
if page_transform is None:
fct_transform = text_transform
else:
def combined_transform(ftpp, filename, content):
text_transform(ftpp, filename, content)
page_transform(ftpp, filename, content)
fct_transform = combined_transform
for project in docs:
fLOG("######################################################################")
for k, v in sorted(project.items()):
fLOG("[publish_documentation] loop {}='{}'".format(k, v))
location = project["local"]
root_local = project["root_local"]
root_web = project["root_web"]
sfile = project["status_file"]
rootw = project["root_web"]
# documentation + setup
fLOG("[publish_documentation] location='{}'".format(location))
ftn = FileTreeNode(root_local)
fftp = FolderTransferFTP(ftn, ftp, sfile, root_web=rootw, fLOG=fLOG, footer_html=footer_html,
content_filter=content_filter, is_binary=is_binary,
text_transform=fct_transform, filter_out=filter_out,
force_allow=force_allow, exc=exc)
fftp.start_transfering(delay=delay)
ftn = FileTreeNode(os.path.join(root_local, ".."),
filter=lambda root, path, f, dir: not dir)
fftp = FolderTransferFTP(ftn, ftp, sfile,
root_web=root_web.replace("helpsphinx", ""), fLOG=fLOG,
footer_html=footer_html, content_filter=content_filter,
is_binary=is_binary, text_transform=fct_transform)
fftp.start_transfering()
ftp.close()
def publish_documentation(docs, ftpsite=None, login=None, password=None,
footer_html=None, content_filter=trigger_on_specific_strings,
is_binary=content_as_binary, force_allow=None,
delay=0.5, exc=False, fLOG=print):
"""
Publishes the documentation and the setups of a python module on a webiste,
it assumes the modules is organized the same way as :epkg:`pyquickhelper`.
@param docs list of dictionaries (see below)
@param ftpsite something like ``ftp.something.``
@param login login
@param password password
@param footer_html append this HTML code to any uploaded page (such a javascript code to count the audience)
@param content_filter filter the content of a file (it raises an exception if the result is None),
appies only on text files
@param is_binary a function to tell if a content of a file is binary or not
@param force_allow a file is not published if it contains something which looks like credentials
except if this string is part of *force_allow*
@param delay delay between file transferring (in average)
@param exc raise exception if not able to transfer
@param fLOG logging function
*docs* is a list of dictionaries which must contain for each folder
to transfer:
- ``local``: local folder
- ``root_local``: local paths will be related to this root
- ``root_web``: prefix to add to the remote paths
- ``status_file``: a file where the function populates the transfered files and some information about them
A local file is composed by ``<local_root>/<relative_path>``, it
will be uploaded to ``<web_root>/<relative_path>``.
"""
params = {"ftpsite": ftpsite,
"login": login,
"password": password}
nbnone = len([v for k, v in params.items() if v is None or len(v) == 0])
if nbnone > 0:
raise ValueError(
"One of the following parameters is not specified:\n{0}".format(params))
nbnone = [v for k, v in params.items() if v is None or len(v) == 0]
if len(nbnone) > 0:
raise Exception("one of the parameters is None:\n" + str(nbnone))
password = params["password"]
login = params["login"]
ftpsite = params["ftpsite"]
filter_out = "([/\\\\]((moduletoc.html)|(blogtoc.html)|(searchbox.html)))|([.]buildinfo)"
ftp = TransferFTP(ftpsite, login, password, fLOG=fLOG)
for project in docs:
fLOG("######################################################################")
for k, v in sorted(project.items()):
fLOG("[publish_documentation] loop {}='{}'".format(k, v))
location = project["local"]
root_local = project["root_local"]
root_web = project["root_web"]
sfile = project["status_file"]
rootw = project["root_web"]
# documentation + setup
fLOG("[publish_documentation] location='{}'".format(location))
ftn = FileTreeNode(root_local)
fftp = FolderTransferFTP(ftn, ftp, sfile, root_web=rootw, fLOG=fLOG, footer_html=footer_html,
content_filter=content_filter, is_binary=is_binary,
text_transform=text_transform, filter_out=filter_out,
force_allow=force_allow, exc=exc)
fftp.start_transfering(delay=delay)
ftn = FileTreeNode(os.path.join(root_local, ".."),
filter=lambda root, path, f, dir: not dir)
fftp = FolderTransferFTP(ftn, ftp, sfile,
root_web=root_web.replace("helpsphinx", ""), fLOG=fLOG,
footer_html=footer_html, content_filter=content_filter,
is_binary=is_binary, text_transform=text_transform)
fftp.start_transfering()
ftp.close()
def publish_documentation(docs,
ftpsite=None,
login=None,
password=None,
key_save="my_password",
footer_html=None,
content_filter=trigger_on_specific_strings,
is_binary=content_as_binary,
force_allow=None,
fLOG=print):
"""
Publishes the documentation and the setups of a python module on a webiste,
it assumes the modules is organized the same way as
`pyquickhelper <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/index.html>`_.
@param docs list of dictionaries (see below)
@param ftpsite something like ``ftp.something.``
@param login login
@param password password
@param key_save see function `open_window_params <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/tkinterquickhelper/funcwin/frame_params.html#tkinterquickhelper.funcwin.frame_params.open_window_params>`_
@param footer_html append this HTML code to any uploaded page (such a javascript code to count the audience)
@param content_filter filter the content of a file (it raises an exception if the result is None),
appies only on text files
@param is_binary a function to tell if a content of a file is binary or not
@param force_allow a file is not published if it contains something which looks like credentials
except if this string is part of *force_allow*
@param fLOG logging function
*docs* is a list of dictionaries which must contain for each folder
to transfer:
- ``local``: local folder
- ``root_local``: local paths will be related to this root
- ``root_web``: prefix to add to the remote paths
- ``status_file``: a file where the function populates the transfered files and some information about them
A local file is composed by ``<local_root>/<relative_path>``, it
will be uploaded to ``<web_root>/<relative_path>``.
If one of the three first parameters is None, the function
will open a popup windows to ask the missing information.
See `open_window_params <http://www.xavierdupre.fr/app/tkinterquickhelper/helpsphinx/tkinterquickhelper/funcwin/frame_params.html#tkinterquickhelper.funcwin.frame_params.open_window_params>`_.
"""
params = {"ftpsite": ftpsite, "login": login, "password": password}
nbnone = len([v for k, v in params.items() if v is None or len(v) == 0])
if nbnone > 0:
from tkinterquickhelper.funcwin import open_window_params
fLOG("retrying to get parameters from users")
for k, v in sorted(params.items()):
fLOG(" {0}={1}".format(k, v))
params = open_window_params(params,
title="Website and Credentials",
help_string="ftp site + login + password",
key_save=key_save)
nbnone = [v for k, v in params.items() if v is None or len(v) == 0]
if len(nbnone) > 0:
raise Exception("one of the parameters is None:\n" + str(nbnone))
password = params["password"]
login = params["login"]
ftpsite = params["ftpsite"]
filter_out = "([/\\\\]((moduletoc.html)|(blogtoc.html)|(searchbox.html)))|([.]buildinfo)"
ftp = TransferFTP(ftpsite, login, password, fLOG=fLOG)
for project in docs:
fLOG(
"######################################################################"
)
for k, v in sorted(project.items()):
fLOG(" {}={}".format(k, v))
location = project["local"]
root_local = project["root_local"]
root_web = project["root_web"]
sfile = project["status_file"]
rootw = project["root_web"]
# documentation + setup
fLOG("-------------------------", location)
ftn = FileTreeNode(root_local)
fftp = FolderTransferFTP(ftn,
ftp,
sfile,
root_web=rootw,
fLOG=fLOG,
footer_html=footer_html,
content_filter=content_filter,
is_binary=is_binary,
text_transform=text_transform,
filter_out=filter_out,
force_allow=force_allow)
fftp.start_transfering()
ftn = FileTreeNode(os.path.join(root_local, ".."),
filter=lambda root, path, f, dir: not dir)
fftp = FolderTransferFTP(ftn,
ftp,
sfile,
root_web=root_web.replace("helpsphinx", ""),
fLOG=fLOG,
footer_html=footer_html,
content_filter=content_filter,
is_binary=is_binary,
text_transform=text_transform)
fftp.start_transfering()
ftp.close()
def test_transfer_ftp_start_transfering(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
import keyring
prefix = "pyquickhelper,"
try:
user = keyring.get_password("web", prefix + "user")
pwd = keyring.get_password("web", prefix + "pwd")
ftpsite = keyring.get_password("web", prefix + "ftp")
except RuntimeError:
user = None
pwd = None
if user is None:
if not is_travis_or_appveyor():
raise Exception("user password is empty, prefix='{0}', username='******'".format(
prefix, get_user()))
return
# Transfering
now = datetime.datetime.now()
temp = get_temp_folder(__file__, "temp_transfer_ftp_true")
with open(os.path.join(temp, "essai.txt"), 'w') as f:
f.write(str(now))
sfile = os.path.join(temp, "status_ut.txt")
ftn = FileTreeNode(temp)
# one
try:
ftp = TransferFTP(ftpsite, user, pwd, fLOG=fLOG)
except ftplib.error_temp as e:
if "421 Home directory not available" in str(e):
return
raise e
except socket.gaierror as ee:
if "Name or service not known" in str(ee):
return
if "getaddrinfo failed" in str(ee):
return
raise ee
web = FolderTransferFTP(ftn, ftp, sfile,
root_web="/www/htdocs/apptest/",
fLOG=fLOG)
done = web.start_transfering(delay=0.1)
ftp.close()
names = [os.path.split(f.filename)[-1] for f in done]
self.assertEqual(names, ['essai.txt'])
# two, same file, should not be done again
ftp = TransferFTP(ftpsite, user, pwd, fLOG=fLOG)
web = FolderTransferFTP(ftn, ftp, sfile,
root_web="/www/htdocs/apptest/",
fLOG=fLOG)
done = web.start_transfering(delay=0.1)
ftp.close()
self.assertEmpty(done)
def publish_documentation(docs, ftpsite=None, login=None, password=None,
key_save="my_password", footer_html=None, content_filter=trigger_on_specific_strings,
is_binary=content_as_binary, force_allow=None, fLOG=print):
"""
publish the documentation and the setups of a python module on a webiste,
it assumes the modules is organized the same way as
`pyquickhelper <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/index.html>`_.
@param docs list of dictionaries (see below)
@param ftpsite something like ``ftp.something.``
@param login login
@param password password
@param key_save see function `open_window_params <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/funcwin/frame_params.html#pyquickhelper.funcwin.frame_params.open_window_params>`_
@param footer_html append this HTML code to any uploaded page (such a javascript code to count the audience)
@param content_filter filter the content of a file (it raises an exception if the result is None),
appies only on text files
@param is_binary a function to tell if a content of a file is binary or not
@param force_allow a file is not published if it contains something which looks like credentials
except if this string is part of *force_allow*
@param fLOG logging function
*docs* is a list of dictionaries which must contain for each folder
to transfer:
- ``local``: local folder
- ``root_local``: local paths will be related to this root
- ``root_web``: prefix to add to the remote paths
- ``status_file``: a file where the function populates the transfered files and some information about them
A local file is composed by ``<local_root>/<relative_path>``, it
will be uploaded to ``<web_root>/<relative_path>``.
If one of the three first parameters is None, the function
will open a popup windows to ask the missing information.
See `open_window_params <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/funcwin/frame_params.html#pyquickhelper.funcwin.frame_params.open_window_params>`_.
"""
params = {"ftpsite": ftpsite,
"login": login,
"password": password}
nbnone = len([v for k, v in params.items() if v is None or len(v) == 0])
if nbnone > 0:
fLOG("retrying to get parameters from users")
for k, v in sorted(params.items()):
fLOG(" {0}={1}".format(k, v))
params = open_window_params(
params,
title="Website and Credentials",
help_string="ftp site + login + password",
key_save=key_save)
nbnone = [v for k, v in params.items() if v is None or len(v) == 0]
if len(nbnone) > 0:
raise Exception("one of the parameters is None:\n" + str(nbnone))
password = params["password"]
login = params["login"]
ftpsite = params["ftpsite"]
filter_out = "[/\\\\]((moduletoc.html)|(blogtoc.html)|(searchbox.html))"
ftp = TransferFTP(ftpsite, login, password, fLOG=fLOG)
for project in docs:
fLOG("######################################################################")
for k, v in sorted(project.items()):
fLOG(" {}={}".format(k, v))
location = project["local"]
root_local = project["root_local"]
root_web = project["root_web"]
sfile = project["status_file"]
rootw = project["root_web"]
# documentation + setup
fLOG("-------------------------", location)
ftn = FileTreeNode(root_local)
fftp = FolderTransferFTP(ftn, ftp, sfile,
root_web=rootw,
fLOG=fLOG,
footer_html=footer_html,
content_filter=content_filter,
is_binary=is_binary,
text_transform=text_transform,
filter_out=filter_out,
force_allow=force_allow)
fftp.start_transfering()
ftn = FileTreeNode(os.path.join(root_local, ".."),
filter=lambda root, path, f, dir: not dir)
fftp = FolderTransferFTP(ftn, ftp, sfile,
root_web=root_web.replace(
"helpsphinx", ""),
fLOG=fLOG,
footer_html=footer_html,
content_filter=content_filter,
is_binary=is_binary,
text_transform=text_transform)
fftp.start_transfering()
ftp.close()
