def unzip(filezip=None, pathdst='', remove_zip=False): """Unzip a ZIP or 7Z file :param filezip: full filename (with path) to ZIP/7Z file :type filezip: string :param pathdst: path to directory where unzip ZIP/7Z file :type pathdst: string :param remove_zip: if True, remove ZIP/7Z file after unzip :type remove_zip: string :return: True or False :rtype: boolean """ if os.path.isfile(filezip): if pathdst == '': pathdst = os.getcwd() ext = os.path.splitext(filezip)[1].lower() if ext == '.zip': # ZIP file with zipfile.ZipFile(filezip, 'r') as zfile: log.log("Files unzip from archive:", 'INFO', 0) for zfile_name in zfile.namelist(): log.log('- ' + zfile_name, 'INFO', 0) zfile_fullpath = os.path.join(pathdst, zfile_name) if os.path.isdir(zfile_name): if not os.path.isdir(zfile_fullpath): os.makedirs(zfile_fullpath) else: if not os.path.isdir(os.path.join(pathdst, os.path.dirname(zfile_name))): os.makedirs(os.path.join(pathdst, os.path.dirname(zfile_name))) data = zfile.read(zfile_name) fp = open(zfile_fullpath, "wb") fp.write(data) fp.close() elif ext == '.7z': # 7ZIP file with open(filezip, 'rb') as fp: archive = py7zlib.Archive7z(fp) for name in archive.getnames(): outfilename = os.path.join(pathdst, name) outdir = os.path.dirname(outfilename) if not os.path.exists(outdir): os.makedirs(outdir) outfile = open(outfilename, 'wb') outfile.write(archive.getmember(name).read()) outfile.close() if remove_zip: # Remove ZIP/7Z file os.remove(filezip) return True return False
def getMetadata(node=[], reader=[]): """Get XML metadata files from CSV file reader :param node: node file configuration values :type node: list :param reader: reader of CSV file :type reader: list """ if node['sftp']: # Define tmp directory for XML files (local_dir) local_dir = os.path.join(node['tmp_dir'], config['MAIN']['xml_tmp_dir']) # Empty remote SFTP directory (only files) helpers.delFiles(node, node['xml_dir']) else: # Define tmp directory for XML files (local_dir) local_dir = node['xml_dir'] # Remove local_dir directory and recreate it (empty) log.log(u'Empty local directory: "' + local_dir + '".', 'INFO', 0) if os.path.isdir(local_dir): shutil.rmtree(local_dir) os.makedirs(local_dir) log.log(u'Get metadata in "' + local_dir + '".', 'INFO', 0) for csv_row in reader: r = row.Row(csv_row, config['COLUMNS']) # Get xml files to local dir log.log(u'Get metadata file: "' + r.getAttr('md_fileid') + '".', 'INFO', 0) r.getXml(local_dir) if node['sftp']: # Put local file to remote directory (SFTP) log.log(u'Put metadata file: "' + r.getAttr('md_fileid') + '".', 'INFO', 0) r.putXml(node, local_dir, node['xml_dir'])
def delFiles(sftp=None, dst=None): """Put local file to remote destination using SFTP :param dst: path directory to delete files :type dst: string """ cnopts = pysftp.CnOpts() cnopts.hostkeys = None with pysftp.Connection(host=sftp['sftp_hostname'], port=sftp['sftp_port'], username=sftp['sftp_username'], private_key_pass=sftp['sftp_private_key_pass'], private_key=sftp['sftp_private_key'], cnopts=cnopts) as sftp: files = sftp.listdir(dst) with sftp.cd(dst): log.log(u'Empty SFTP directory: "' + dst + '".', 'INFO', 0) for file in files: log.log(u'Remove SFTP file: "' + file + '".', 'INFO', 0) sftp.remove(file)
def getFile(src=None, dst=None, verify_certificate=True): """Get local or remote file :param src: URL or full filename (with path) to get file (remote source) :type src: string :param dst: full filename (with path) to copy source file in local destination directory :type dst: string :param verify_certificate: check certificate validation :type verify_certificate: boolean """ log.log('Copy ' + src + ' to ' + dst + ' file.', 'INFO', 0) if src.startswith('http'): r = requests.get(src, verify=verify_certificate) # Save CSV file in temp directory with open(dst, 'w') as dst_file: dst_file.write(r.text.encode('iso-8859-1')) else: # Copy CSV file to tmp directory shutil.copy(src, dst)
def createLayer(self, tmp_dir=None, cat=None, ws=None): """Create a layer from row values :param tmp_dir: tmp_dir to unzip file and get data :type tmp_dir: string :param cat: geoserver catalog object of gs-config :type cat: object :param ws: geoserver workspace object of gs-config :type ws: object :return: True or False :rtype: boolean """ if self.layer_name and tmp_dir and cat: # Get and unzip data file of DATA_FILEZIP CSV column pathfile = helpers.getUnzipFile(self.data_filezip, tmp_dir, True, self.disable_ssl_verify) # If pathfile exists (zip file has been unzip) if pathfile: # Create style to Geoserver if necessary or overwrite it if self.style_name and os.path.isfile(self.sld_file): log.log('Create style ' + self.style_name + ' in Geoserver.', 'INFO', 0) Style(self.style_name, self.sld_file).createStyle(cat) # Add layer to Geoserver log.log('Create or overwrite data layer ' + self.layer_name + ' in Geoserver.', 'INFO', 0) data = geoserver.util.shapefile_and_friends(pathfile) cat.create_featurestore(name=self.layer_name, data=data, workspace=ws, overwrite=True) # Update layer info log.log('Update metadata layer ' + self.layer_name + ' in Geoserver.', 'INFO', 0) self.updateLayer(cat, ws) return True return False
def setUp(self): self.driver = webdriver.Firefox() self.url = 'http://192.168.1.79:8080' # self.url = 'http://120.27.198.114:8989/LdxSmart-0.1' # self. client = MongoClient('120.27.198.114', 27017, connect=False) self. client = MongoClient('192.168.1.79', 27017, connect=False) self.name = 'SZCR' self.passwd = '123456' self.title = u'用户登录_EXSOFT' # self.gc_title = u'收货' self.login_page = page_login.loginIn(self.driver) self.gc = page_gc.GCPage(self.driver) self.mylog = log.log()
def getUnzipFile(url=False, tmp_dir=False, remove_zip=False, verify_certificate=True): """Get and unzip a file :param url: URL or full filename (with path) to ZIP file :type url: string :param verify_certificate: check certificate validation :type verify_certificate: boolean :param tmp_dir: path to directory where unzip ZIP file :type tmp_dir: string :param remove_zip: if True, remove ZIP file after unzip :type remove_zip: string :return: path to unzip files or False :rtype: string or boolean """ log.log('Get and unzip ' + url + ' file in ' + tmp_dir + '.', 'INFO', 0) if url: dirname, basename = os.path.split(url) filename, ext = os.path.splitext(basename) if tmp_dir and ext in ['.zip', '.7zip']: tmp_data_dir = os.path.join(tmp_dir, filename) tmp_data_file = os.path.join(tmp_data_dir, basename) # Create tmp directory to get layer data zip if not os.path.exists(tmp_data_dir): os.makedirs(tmp_data_dir) # Get and copy zip file if url.startswith('http'): # Remote zip file log.log("Get remote ZIP file.", 'INFO', 0) response = requests.get(url, stream=True, verify=verify_certificate) with open(tmp_data_file, 'wb') as out_file: shutil.copyfileobj(response.raw, out_file) del response else: # Local zip file log.log("Get local ZIP file.", 'INFO', 0) shutil.copy(url, tmp_data_file) # Unzip file (delete zip file if remove-zip is True) unzip(tmp_data_file, tmp_data_dir, remove_zip) # Return path to unzip files return os.path.join(tmp_data_dir, filename) return False
def getData(node=[], reader=[], tmp_csv_filepath=''): """Get data ZIP/7Z files from CSV file reader :param node: node file configuration values :type node: list :param reader: reader of CSV file :type reader: list :param tmp_csv_filepath: fullpath of tmp file CSV :type tmp_csv_filepath: string """ # Connexion to Geoserver log.log(u'Connexion to Geoserver workspace ' + node['gs_workspace'], 'INFO', 0) cat = geoserver.catalog.Catalog(node['gs_url'] + 'rest', username=node['gs_login'], password=node['gs_pwd'], disable_ssl_certificate_validation=node['gs_disable_certificate_validation']) ws = cat.get_workspace(node['gs_workspace']) if ws is None: log.log(u'Create workspace ' + node['gs_workspace'], 'INFO', 0) ws = cat.create_workspace(node['gs_workspace'], node['gs_url'] + node['gs_workspace']) # Manage layer creation in geoserver if not os.path.isfile(tmp_csv_filepath): # First harvesting for this node for csv_row in reader: r = row.Row(csv_row, config['COLUMNS']) log.log('Create layer ' + r.getAttr('layer_name') + ' or overwrite it if exists in Geoserver.', 'INFO', 0) r.createLayer(node['tmp_dir'], cat, ws) else: # Read CSV tmp file with open(tmp_csv_filepath, 'rt') as tmp_file: tmp_reader = list(csv.DictReader(tmp_file)) # Init del layers del_layers = [] for csv_row_tmp in tmp_reader: tmp_r = row.Row(csv_row_tmp, config['COLUMNS']) del_layers.append(tmp_r.getAttr('layer_name')) # Manage layers for csv_row in reader: r = row.Row(csv_row, config['COLUMNS']) add = True for csv_row_tmp in tmp_reader: tmp_r = row.Row(csv_row_tmp, config['COLUMNS']) if r.getAttr('layer_name') == tmp_r.getAttr('layer_name'): # Layer exists in tmp_file: not need to create it add = False del_layers.remove(tmp_r.getAttr('layer_name')) if r.getAttr('date') != config['MAIN']['always_sync'] and r.getAttr('date') != tmp_r.getAttr('date'): # Update layer => data + metadata = create layer with overwrite log.log('Update layer ' + r.getAttr('layer_name') + ': overwrite data and metadata.', 'INFO', 0) r.createLayer(node['tmp_dir'], cat, ws) else: log.log('Layer ' + r.getAttr('layer_name') + ' not modified.', 'INFO', 0) if add: # layer is in reader but not in tmp_reader log.log('Create layer ' + r.getAttr('layer_name') + ' or overwrite it if exists in Geoserver.', 'INFO', 0) r.createLayer(node['tmp_dir'], cat, ws) for csv_row_del in del_layers: # Delete layers of del_layers dictionary del_r = row.Row(csv_row_del, config['COLUMNS']) log.log('Delete layer ' + del_r.getAttr('layer_name') + '.', 'INFO', 0) del_r.deleteLayer(cat)
def run(): # if args.file: # pass # else: # Get list of nodes files from config node_dir nodes_files = [f for f in os.listdir(config['MAIN']['nodes_dir']) if os.path.isfile(os.path.join(config['MAIN']['nodes_dir'], f))] count_file = 1 for nodes_file in nodes_files: log.log('', 'INFO', 0) log.log(color.OKGREEN + u'*' * 80 + color.ENDC, 'INFO', 0) log.log(color.OKGREEN + u'File ' + str(count_file) + '/' + str(len(nodes_files)) + ': ' + nodes_file + '.' + color.ENDC, 'INFO', 0) if nodes_file.startswith('_'): log.log(u'File skipped.', 'INFO', 0) else: file_path = os.path.join(config['MAIN']['nodes_dir'], nodes_file) with open(file_path, 'r') as json_data: data = json.load(json_data) log.log(color.BOLD + u'Organisme: ' + data['organisme'] + color.ENDC, 'INFO', 0) count_node = 1 for node in data['nodes']: log.log('', 'INFO', 0) log.log(color.OKBLUE + u'-' * 80 + color.ENDC, 'INFO', 0) log.log(color.OKBLUE + u'Node ' + str(count_node) + '/' + str(len(data['nodes'])) + color.ENDC, 'INFO', 0) log.log(u'Description: ' + node['description'], 'INFO', 0) if node['active'] == '0': log.log(u'Node disabled.', 'INFO', 0) else: # Log node informations log.log(u'Active: ' + node['active'], 'INFO', 0) log.log(u'CSV file: ' + os.path.join(node['src_csv_path'], node['src_csv_file']), 'INFO', 0) log.log(u'XML directory: ' + node['xml_dir'], 'INFO', 0) log.log(u'TMP directory: ' + node['tmp_dir'], 'INFO', 0) # Full path to CSV file csv_filepath = os.path.join(node['tmp_dir'], node['src_csv_file']) # Full path to CSV temp file tmp_csv_file = 'tmp_' + node['src_csv_file'] tmp_csv_filepath = os.path.join(node['tmp_dir'], tmp_csv_file) # Create tmp_dir if not os.path.isdir(node['tmp_dir']): os.makedirs(node['tmp_dir']) # Create xml_tmp_dir xml_tmp_dir = os.path.join(node['tmp_dir'], config['MAIN']['xml_tmp_dir']) if not os.path.isdir(xml_tmp_dir): os.makedirs(xml_tmp_dir) log.log(u'Start: ' + str(time.strftime("%Y-%m-%d %H:%M:%S")), 'INFO', 0) # Get file from path or URL helpers.getFile(os.path.join(node['src_csv_path'], node['src_csv_file']), csv_filepath) # Read CSV saved file with open(csv_filepath, 'rt') as csv_file: reader = list(csv.DictReader(csv_file)) # metadata_active = 0/1 log.log(u'Metadata active: ' + str(config['MAIN']['metadata_active']), 'INFO', 0) if config['MAIN']['metadata_active']: getMetadata(node, reader) log.log(u'Data active: ' + str(config['MAIN']['data_active']), 'INFO', 0) # data_active = 0/1 if config['MAIN']['data_active']: getData(node, reader, tmp_csv_filepath) # Copy CSV file to tmp file shutil.copy(csv_filepath, tmp_csv_filepath) log.log(u'End: ' + str(time.strftime("%Y-%m-%d %H:%M:%S")), 'INFO', 0) count_node += 1 # nodes_file.close() count_file += 1
def __init__(self, selenium_driver): # type: (object, object, object) -> object self.driver = selenium_driver self.mylog = log.log()