def _connect(self):
self.ftp = ftputil.FTPHost(
self.base_url,
"anonymous",
"od-database",
session_factory=session_factory(use_passive_mode=False))
self.ftp._session.timeout = 40
def get_conn(self):
if ftputil is None:
raise ImportError("ftputil must be installed to use the FtpHook")
if self._conn is None:
config = self.get_connection(self._conn_id)
secure = config.extra_dejson.get('tls', False)
base_class = ftplib.FTP_TLS if secure else ftplib.FTP
session_factory = ftp_session.session_factory(
base_class=base_class,
port=config.port or 21,
encrypt_data_channel=secure)
self._conn = ftputil.FTPHost(
config.host,
config.login,
config.password,
session_factory=session_factory)
return self._conn
import ftputil
import requests
from ftputil import session
from scrapy import Selector
from .mappings import LICENSES, LICENSE_TEXTS
RE_FOR_THE = re.compile(r'\b(?:for|on behalf of|representing)\b',
re.IGNORECASE)
INST_PHRASES = [
'for the development',
]
ftp_session_factory = session.session_factory(base_class=ftplib.FTP,
port=21,
encrypt_data_channel=False,
use_passive_mode=True)
def unzip_files(filename, target_folder, type=None):
"""Unzip files (XML only) into target folder.
:param type: filters files that ends with this value. E.g. for xml files '.xml'
"""
z = ZipFile(filename)
xml_files = []
for filename in z.namelist():
if not type or type and filename.endswith(type):
absolute_path = os.path.join(target_folder, filename)
if not os.path.exists(absolute_path):
z.extract(filename, target_folder)
xml_files.append(absolute_path)