def unzip(filezip=None, pathdst='', remove_zip=False):
"""Unzip a ZIP or 7Z file
:param filezip: full filename (with path) to ZIP/7Z file
:type filezip: string
:param pathdst: path to directory where unzip ZIP/7Z file
:type pathdst: string
:param remove_zip: if True, remove ZIP/7Z file after unzip
:type remove_zip: string
:return: True or False
:rtype: boolean
"""
if os.path.isfile(filezip):
if pathdst == '':
pathdst = os.getcwd()
ext = os.path.splitext(filezip)[1].lower()
if ext == '.zip':
# ZIP file
with zipfile.ZipFile(filezip, 'r') as zfile:
log.log("Files unzip from archive:", 'INFO', 0)
for zfile_name in zfile.namelist():
log.log('- ' + zfile_name, 'INFO', 0)
zfile_fullpath = os.path.join(pathdst, zfile_name)
if os.path.isdir(zfile_name):
if not os.path.isdir(zfile_fullpath):
os.makedirs(zfile_fullpath)
else:
if not os.path.isdir(os.path.join(pathdst, os.path.dirname(zfile_name))):
os.makedirs(os.path.join(pathdst, os.path.dirname(zfile_name)))
data = zfile.read(zfile_name)
fp = open(zfile_fullpath, "wb")
fp.write(data)
fp.close()
elif ext == '.7z':
# 7ZIP file
with open(filezip, 'rb') as fp:
archive = py7zlib.Archive7z(fp)
for name in archive.getnames():
outfilename = os.path.join(pathdst, name)
outdir = os.path.dirname(outfilename)
if not os.path.exists(outdir):
os.makedirs(outdir)
outfile = open(outfilename, 'wb')
outfile.write(archive.getmember(name).read())
outfile.close()
if remove_zip:
# Remove ZIP/7Z file
os.remove(filezip)
return True
return False
def getMetadata(node=[], reader=[]):
"""Get XML metadata files from CSV file reader
:param node: node file configuration values
:type node: list
:param reader: reader of CSV file
:type reader: list
"""
if node['sftp']:
# Define tmp directory for XML files (local_dir)
local_dir = os.path.join(node['tmp_dir'], config['MAIN']['xml_tmp_dir'])
# Empty remote SFTP directory (only files)
helpers.delFiles(node, node['xml_dir'])
else:
# Define tmp directory for XML files (local_dir)
local_dir = node['xml_dir']
# Remove local_dir directory and recreate it (empty)
log.log(u'Empty local directory: "' + local_dir + '".', 'INFO', 0)
if os.path.isdir(local_dir):
shutil.rmtree(local_dir)
os.makedirs(local_dir)
log.log(u'Get metadata in "' + local_dir + '".', 'INFO', 0)
for csv_row in reader:
r = row.Row(csv_row, config['COLUMNS'])
# Get xml files to local dir
log.log(u'Get metadata file: "' + r.getAttr('md_fileid') + '".', 'INFO', 0)
r.getXml(local_dir)
if node['sftp']:
# Put local file to remote directory (SFTP)
log.log(u'Put metadata file: "' + r.getAttr('md_fileid') + '".', 'INFO', 0)
r.putXml(node, local_dir, node['xml_dir'])
def delFiles(sftp=None, dst=None):
"""Put local file to remote destination using SFTP
:param dst: path directory to delete files
:type dst: string
"""
cnopts = pysftp.CnOpts()
cnopts.hostkeys = None
with pysftp.Connection(host=sftp['sftp_hostname'], port=sftp['sftp_port'], username=sftp['sftp_username'], private_key_pass=sftp['sftp_private_key_pass'], private_key=sftp['sftp_private_key'], cnopts=cnopts) as sftp:
files = sftp.listdir(dst)
with sftp.cd(dst):
log.log(u'Empty SFTP directory: "' + dst + '".', 'INFO', 0)
for file in files:
log.log(u'Remove SFTP file: "' + file + '".', 'INFO', 0)
sftp.remove(file)
def getFile(src=None, dst=None, verify_certificate=True):
"""Get local or remote file
:param src: URL or full filename (with path) to get file (remote source)
:type src: string
:param dst: full filename (with path) to copy source file in local destination directory
:type dst: string
:param verify_certificate: check certificate validation
:type verify_certificate: boolean
"""
log.log('Copy ' + src + ' to ' + dst + ' file.', 'INFO', 0)
if src.startswith('http'):
r = requests.get(src, verify=verify_certificate)
# Save CSV file in temp directory
with open(dst, 'w') as dst_file:
dst_file.write(r.text.encode('iso-8859-1'))
else:
# Copy CSV file to tmp directory
shutil.copy(src, dst)
def createLayer(self, tmp_dir=None, cat=None, ws=None):
"""Create a layer from row values
:param tmp_dir: tmp_dir to unzip file and get data
:type tmp_dir: string
:param cat: geoserver catalog object of gs-config
:type cat: object
:param ws: geoserver workspace object of gs-config
:type ws: object
:return: True or False
:rtype: boolean
"""
if self.layer_name and tmp_dir and cat:
# Get and unzip data file of DATA_FILEZIP CSV column
pathfile = helpers.getUnzipFile(self.data_filezip, tmp_dir, True, self.disable_ssl_verify)
# If pathfile exists (zip file has been unzip)
if pathfile:
# Create style to Geoserver if necessary or overwrite it
if self.style_name and os.path.isfile(self.sld_file):
log.log('Create style ' + self.style_name + ' in Geoserver.', 'INFO', 0)
Style(self.style_name, self.sld_file).createStyle(cat)
# Add layer to Geoserver
log.log('Create or overwrite data layer ' + self.layer_name + ' in Geoserver.', 'INFO', 0)
data = geoserver.util.shapefile_and_friends(pathfile)
cat.create_featurestore(name=self.layer_name, data=data, workspace=ws, overwrite=True)
# Update layer info
log.log('Update metadata layer ' + self.layer_name + ' in Geoserver.', 'INFO', 0)
self.updateLayer(cat, ws)
return True
return False
def setUp(self):
self.driver = webdriver.Firefox()
self.url = 'http://192.168.1.79:8080'
# self.url = 'http://120.27.198.114:8989/LdxSmart-0.1'
# self. client = MongoClient('120.27.198.114', 27017, connect=False)
self. client = MongoClient('192.168.1.79', 27017, connect=False)
self.name = 'SZCR'
self.passwd = '123456'
self.title = u'用户登录_EXSOFT'
# self.gc_title = u'收货'
self.login_page = page_login.loginIn(self.driver)
self.gc = page_gc.GCPage(self.driver)
self.mylog = log.log()
def getUnzipFile(url=False, tmp_dir=False, remove_zip=False, verify_certificate=True):
"""Get and unzip a file
:param url: URL or full filename (with path) to ZIP file
:type url: string
:param verify_certificate: check certificate validation
:type verify_certificate: boolean
:param tmp_dir: path to directory where unzip ZIP file
:type tmp_dir: string
:param remove_zip: if True, remove ZIP file after unzip
:type remove_zip: string
:return: path to unzip files or False
:rtype: string or boolean
"""
log.log('Get and unzip ' + url + ' file in ' + tmp_dir + '.', 'INFO', 0)
if url:
dirname, basename = os.path.split(url)
filename, ext = os.path.splitext(basename)
if tmp_dir and ext in ['.zip', '.7zip']:
tmp_data_dir = os.path.join(tmp_dir, filename)
tmp_data_file = os.path.join(tmp_data_dir, basename)
# Create tmp directory to get layer data zip
if not os.path.exists(tmp_data_dir):
os.makedirs(tmp_data_dir)
# Get and copy zip file
if url.startswith('http'):
# Remote zip file
log.log("Get remote ZIP file.", 'INFO', 0)
response = requests.get(url, stream=True, verify=verify_certificate)
with open(tmp_data_file, 'wb') as out_file:
shutil.copyfileobj(response.raw, out_file)
del response
else:
# Local zip file
log.log("Get local ZIP file.", 'INFO', 0)
shutil.copy(url, tmp_data_file)
# Unzip file (delete zip file if remove-zip is True)
unzip(tmp_data_file, tmp_data_dir, remove_zip)
# Return path to unzip files
return os.path.join(tmp_data_dir, filename)
return False
def getData(node=[], reader=[], tmp_csv_filepath=''):
"""Get data ZIP/7Z files from CSV file reader
:param node: node file configuration values
:type node: list
:param reader: reader of CSV file
:type reader: list
:param tmp_csv_filepath: fullpath of tmp file CSV
:type tmp_csv_filepath: string
"""
# Connexion to Geoserver
log.log(u'Connexion to Geoserver workspace ' + node['gs_workspace'], 'INFO', 0)
cat = geoserver.catalog.Catalog(node['gs_url'] + 'rest', username=node['gs_login'], password=node['gs_pwd'], disable_ssl_certificate_validation=node['gs_disable_certificate_validation'])
ws = cat.get_workspace(node['gs_workspace'])
if ws is None:
log.log(u'Create workspace ' + node['gs_workspace'], 'INFO', 0)
ws = cat.create_workspace(node['gs_workspace'], node['gs_url'] + node['gs_workspace'])
# Manage layer creation in geoserver
if not os.path.isfile(tmp_csv_filepath):
# First harvesting for this node
for csv_row in reader:
r = row.Row(csv_row, config['COLUMNS'])
log.log('Create layer ' + r.getAttr('layer_name') + ' or overwrite it if exists in Geoserver.', 'INFO', 0)
r.createLayer(node['tmp_dir'], cat, ws)
else:
# Read CSV tmp file
with open(tmp_csv_filepath, 'rt') as tmp_file:
tmp_reader = list(csv.DictReader(tmp_file))
# Init del layers
del_layers = []
for csv_row_tmp in tmp_reader:
tmp_r = row.Row(csv_row_tmp, config['COLUMNS'])
del_layers.append(tmp_r.getAttr('layer_name'))
# Manage layers
for csv_row in reader:
r = row.Row(csv_row, config['COLUMNS'])
add = True
for csv_row_tmp in tmp_reader:
tmp_r = row.Row(csv_row_tmp, config['COLUMNS'])
if r.getAttr('layer_name') == tmp_r.getAttr('layer_name'):
# Layer exists in tmp_file: not need to create it
add = False
del_layers.remove(tmp_r.getAttr('layer_name'))
if r.getAttr('date') != config['MAIN']['always_sync'] and r.getAttr('date') != tmp_r.getAttr('date'):
# Update layer => data + metadata = create layer with overwrite
log.log('Update layer ' + r.getAttr('layer_name') + ': overwrite data and metadata.', 'INFO', 0)
r.createLayer(node['tmp_dir'], cat, ws)
else:
log.log('Layer ' + r.getAttr('layer_name') + ' not modified.', 'INFO', 0)
if add:
# layer is in reader but not in tmp_reader
log.log('Create layer ' + r.getAttr('layer_name') + ' or overwrite it if exists in Geoserver.', 'INFO', 0)
r.createLayer(node['tmp_dir'], cat, ws)
for csv_row_del in del_layers:
# Delete layers of del_layers dictionary
del_r = row.Row(csv_row_del, config['COLUMNS'])
log.log('Delete layer ' + del_r.getAttr('layer_name') + '.', 'INFO', 0)
del_r.deleteLayer(cat)
def run():
# if args.file:
# pass
# else:
# Get list of nodes files from config node_dir
nodes_files = [f for f in os.listdir(config['MAIN']['nodes_dir']) if os.path.isfile(os.path.join(config['MAIN']['nodes_dir'], f))]
count_file = 1
for nodes_file in nodes_files:
log.log('', 'INFO', 0)
log.log(color.OKGREEN + u'*' * 80 + color.ENDC, 'INFO', 0)
log.log(color.OKGREEN + u'File ' + str(count_file) + '/' + str(len(nodes_files)) + ': ' + nodes_file + '.' + color.ENDC, 'INFO', 0)
if nodes_file.startswith('_'):
log.log(u'File skipped.', 'INFO', 0)
else:
file_path = os.path.join(config['MAIN']['nodes_dir'], nodes_file)
with open(file_path, 'r') as json_data:
data = json.load(json_data)
log.log(color.BOLD + u'Organisme: ' + data['organisme'] + color.ENDC, 'INFO', 0)
count_node = 1
for node in data['nodes']:
log.log('', 'INFO', 0)
log.log(color.OKBLUE + u'-' * 80 + color.ENDC, 'INFO', 0)
log.log(color.OKBLUE + u'Node ' + str(count_node) + '/' + str(len(data['nodes'])) + color.ENDC, 'INFO', 0)
log.log(u'Description: ' + node['description'], 'INFO', 0)
if node['active'] == '0':
log.log(u'Node disabled.', 'INFO', 0)
else:
# Log node informations
log.log(u'Active: ' + node['active'], 'INFO', 0)
log.log(u'CSV file: ' + os.path.join(node['src_csv_path'], node['src_csv_file']), 'INFO', 0)
log.log(u'XML directory: ' + node['xml_dir'], 'INFO', 0)
log.log(u'TMP directory: ' + node['tmp_dir'], 'INFO', 0)
# Full path to CSV file
csv_filepath = os.path.join(node['tmp_dir'], node['src_csv_file'])
# Full path to CSV temp file
tmp_csv_file = 'tmp_' + node['src_csv_file']
tmp_csv_filepath = os.path.join(node['tmp_dir'], tmp_csv_file)
# Create tmp_dir
if not os.path.isdir(node['tmp_dir']):
os.makedirs(node['tmp_dir'])
# Create xml_tmp_dir
xml_tmp_dir = os.path.join(node['tmp_dir'], config['MAIN']['xml_tmp_dir'])
if not os.path.isdir(xml_tmp_dir):
os.makedirs(xml_tmp_dir)
log.log(u'Start: ' + str(time.strftime("%Y-%m-%d %H:%M:%S")), 'INFO', 0)
# Get file from path or URL
helpers.getFile(os.path.join(node['src_csv_path'], node['src_csv_file']), csv_filepath)
# Read CSV saved file
with open(csv_filepath, 'rt') as csv_file:
reader = list(csv.DictReader(csv_file))
# metadata_active = 0/1
log.log(u'Metadata active: ' + str(config['MAIN']['metadata_active']), 'INFO', 0)
if config['MAIN']['metadata_active']:
getMetadata(node, reader)
log.log(u'Data active: ' + str(config['MAIN']['data_active']), 'INFO', 0)
# data_active = 0/1
if config['MAIN']['data_active']:
getData(node, reader, tmp_csv_filepath)
# Copy CSV file to tmp file
shutil.copy(csv_filepath, tmp_csv_filepath)
log.log(u'End: ' + str(time.strftime("%Y-%m-%d %H:%M:%S")), 'INFO', 0)
count_node += 1
# nodes_file.close()
count_file += 1
def __init__(self, selenium_driver):
# type: (object, object, object) -> object
self.driver = selenium_driver
self.mylog = log.log()
