예제 #1
0
def unzip(filezip=None, pathdst='', remove_zip=False):
    """Unzip a ZIP or 7Z file

    :param filezip: full filename (with path) to ZIP/7Z file
    :type filezip: string
    :param pathdst: path to directory where unzip ZIP/7Z file
    :type pathdst: string
    :param remove_zip: if True, remove ZIP/7Z file after unzip
    :type remove_zip: string
    :return: True or False
    :rtype: boolean

    """
    if os.path.isfile(filezip):
        if pathdst == '':
            pathdst = os.getcwd()
        ext = os.path.splitext(filezip)[1].lower()
        if ext == '.zip':
            # ZIP file
            with zipfile.ZipFile(filezip, 'r') as zfile:
                log.log("Files unzip from archive:", 'INFO', 0)
                for zfile_name in zfile.namelist():
                    log.log('- ' + zfile_name, 'INFO', 0)
                    zfile_fullpath = os.path.join(pathdst, zfile_name)
                    if os.path.isdir(zfile_name):
                        if not os.path.isdir(zfile_fullpath):
                            os.makedirs(zfile_fullpath)
                    else:
                        if not os.path.isdir(os.path.join(pathdst, os.path.dirname(zfile_name))):
                            os.makedirs(os.path.join(pathdst, os.path.dirname(zfile_name)))
                        data = zfile.read(zfile_name)
                        fp = open(zfile_fullpath, "wb")
                        fp.write(data)
                        fp.close()
        elif ext == '.7z':
            # 7ZIP file
            with open(filezip, 'rb') as fp:
                archive = py7zlib.Archive7z(fp)
                for name in archive.getnames():
                    outfilename = os.path.join(pathdst, name)
                    outdir = os.path.dirname(outfilename)
                    if not os.path.exists(outdir):
                        os.makedirs(outdir)
                    outfile = open(outfilename, 'wb')
                    outfile.write(archive.getmember(name).read())
                    outfile.close()
        if remove_zip:
            # Remove ZIP/7Z file
            os.remove(filezip)
        return True
    return False
예제 #2
0
def getMetadata(node=[], reader=[]):
    """Get XML metadata files from CSV file reader

    :param node: node file configuration values
    :type node: list
    :param reader: reader of CSV file
    :type reader: list
    """
    if node['sftp']:
        # Define tmp directory for XML files (local_dir)
        local_dir = os.path.join(node['tmp_dir'], config['MAIN']['xml_tmp_dir'])
        # Empty remote SFTP directory (only files)
        helpers.delFiles(node, node['xml_dir'])
    else:
        # Define tmp directory for XML files (local_dir)
        local_dir = node['xml_dir']
    # Remove local_dir directory and recreate it (empty)
    log.log(u'Empty local directory: "' + local_dir + '".', 'INFO', 0)
    if os.path.isdir(local_dir):
        shutil.rmtree(local_dir)
    os.makedirs(local_dir)
    log.log(u'Get metadata in "' + local_dir + '".', 'INFO', 0)
    for csv_row in reader:
        r = row.Row(csv_row, config['COLUMNS'])
        # Get xml files to local dir
        log.log(u'Get metadata file: "' + r.getAttr('md_fileid') + '".', 'INFO', 0)
        r.getXml(local_dir)
        if node['sftp']:
            # Put local file to remote directory (SFTP)
            log.log(u'Put metadata file: "' + r.getAttr('md_fileid') + '".', 'INFO', 0)
            r.putXml(node, local_dir, node['xml_dir'])
예제 #3
0
def delFiles(sftp=None, dst=None):
    """Put local file to remote destination using SFTP

    :param dst: path directory to delete files
    :type dst: string

    """
    cnopts = pysftp.CnOpts()
    cnopts.hostkeys = None

    with pysftp.Connection(host=sftp['sftp_hostname'], port=sftp['sftp_port'], username=sftp['sftp_username'], private_key_pass=sftp['sftp_private_key_pass'], private_key=sftp['sftp_private_key'], cnopts=cnopts) as sftp:
        files = sftp.listdir(dst)
        with sftp.cd(dst):
            log.log(u'Empty SFTP directory: "' + dst + '".', 'INFO', 0)
            for file in files:
                log.log(u'Remove SFTP file: "' + file + '".', 'INFO', 0)
                sftp.remove(file)
예제 #4
0
def getFile(src=None, dst=None, verify_certificate=True):
    """Get local or remote file

    :param src: URL or full filename (with path) to get file (remote source)
    :type src: string
    :param dst: full filename (with path) to copy source file in local destination directory
    :type dst: string
    :param verify_certificate: check certificate validation
    :type verify_certificate: boolean

    """
    log.log('Copy ' + src + ' to ' + dst + ' file.', 'INFO', 0)
    if src.startswith('http'):
        r = requests.get(src, verify=verify_certificate)
        # Save CSV file in temp directory
        with open(dst, 'w') as dst_file:
            dst_file.write(r.text.encode('iso-8859-1'))
    else:
        # Copy CSV file to tmp directory
        shutil.copy(src, dst)
예제 #5
0
파일: row.py 프로젝트: cigalsace/getData
    def createLayer(self, tmp_dir=None, cat=None, ws=None):
        """Create a layer from row values

        :param tmp_dir: tmp_dir to unzip file and get data
        :type tmp_dir: string
        :param cat: geoserver catalog object of gs-config
        :type cat: object
        :param ws: geoserver workspace object of gs-config
        :type ws: object
        :return: True or False
        :rtype: boolean

        """
        if self.layer_name and tmp_dir and cat:
            # Get and unzip data file of DATA_FILEZIP CSV column
            pathfile = helpers.getUnzipFile(self.data_filezip, tmp_dir, True, self.disable_ssl_verify)
            # If pathfile exists (zip file has been unzip)
            if pathfile:
                # Create style to Geoserver if necessary or overwrite it
                if self.style_name and os.path.isfile(self.sld_file):
                    log.log('Create style ' + self.style_name + ' in Geoserver.', 'INFO', 0)
                    Style(self.style_name, self.sld_file).createStyle(cat)
                # Add layer to Geoserver
                log.log('Create or overwrite data layer ' + self.layer_name + ' in Geoserver.', 'INFO', 0)
                data = geoserver.util.shapefile_and_friends(pathfile)
                cat.create_featurestore(name=self.layer_name, data=data, workspace=ws, overwrite=True)
            # Update layer info
            log.log('Update metadata layer ' + self.layer_name + ' in Geoserver.', 'INFO', 0)
            self.updateLayer(cat, ws)
            return True
        return False
예제 #6
0
 def setUp(self):
     self.driver = webdriver.Firefox()
     self.url = 'http://192.168.1.79:8080'
     # self.url = 'http://120.27.198.114:8989/LdxSmart-0.1'
     # self. client = MongoClient('120.27.198.114', 27017, connect=False)
     self. client = MongoClient('192.168.1.79', 27017, connect=False)
     self.name = 'SZCR'
     self.passwd = '123456'
     self.title = u'用户登录_EXSOFT'
     # self.gc_title = u'收货'
     self.login_page = page_login.loginIn(self.driver)
     self.gc = page_gc.GCPage(self.driver)
     self.mylog = log.log()
예제 #7
0
def getUnzipFile(url=False, tmp_dir=False, remove_zip=False, verify_certificate=True):
    """Get and unzip a file

    :param url: URL or full filename (with path) to ZIP file
    :type url: string
    :param verify_certificate: check certificate validation
    :type verify_certificate: boolean
    :param tmp_dir: path to directory where unzip ZIP file
    :type tmp_dir: string
    :param remove_zip: if True, remove ZIP file after unzip
    :type remove_zip: string
    :return: path to unzip files or False
    :rtype: string or boolean

    """
    log.log('Get and unzip ' + url + ' file in ' + tmp_dir + '.', 'INFO', 0)
    if url:
        dirname, basename = os.path.split(url)
        filename, ext = os.path.splitext(basename)

        if tmp_dir and ext in ['.zip', '.7zip']:
            tmp_data_dir = os.path.join(tmp_dir, filename)
            tmp_data_file = os.path.join(tmp_data_dir, basename)

            # Create tmp directory to get layer data zip
            if not os.path.exists(tmp_data_dir):
                os.makedirs(tmp_data_dir)

            # Get and copy zip file
            if url.startswith('http'):
                # Remote zip file
                log.log("Get remote ZIP file.", 'INFO', 0)
                response = requests.get(url, stream=True, verify=verify_certificate)
                with open(tmp_data_file, 'wb') as out_file:
                    shutil.copyfileobj(response.raw, out_file)
                del response
            else:
                # Local zip file
                log.log("Get local ZIP file.", 'INFO', 0)
                shutil.copy(url, tmp_data_file)
            # Unzip file (delete zip file if remove-zip is True)
            unzip(tmp_data_file, tmp_data_dir, remove_zip)
            # Return path to unzip files
            return os.path.join(tmp_data_dir, filename)
    return False
예제 #8
0
def getData(node=[], reader=[], tmp_csv_filepath=''):
    """Get data ZIP/7Z files from CSV file reader

    :param node: node file configuration values
    :type node: list
    :param reader: reader of CSV file
    :type reader: list
    :param tmp_csv_filepath: fullpath of tmp file CSV
    :type tmp_csv_filepath: string
    """
    # Connexion to Geoserver
    log.log(u'Connexion to Geoserver workspace ' + node['gs_workspace'], 'INFO', 0)
    cat = geoserver.catalog.Catalog(node['gs_url'] + 'rest', username=node['gs_login'], password=node['gs_pwd'], disable_ssl_certificate_validation=node['gs_disable_certificate_validation'])
    ws = cat.get_workspace(node['gs_workspace'])
    if ws is None:
        log.log(u'Create workspace ' + node['gs_workspace'], 'INFO', 0)
        ws = cat.create_workspace(node['gs_workspace'], node['gs_url'] + node['gs_workspace'])
    # Manage layer creation in geoserver
    if not os.path.isfile(tmp_csv_filepath):
        # First harvesting for this node
        for csv_row in reader:
            r = row.Row(csv_row, config['COLUMNS'])
            log.log('Create layer ' + r.getAttr('layer_name') + ' or overwrite it if exists in Geoserver.', 'INFO', 0)
            r.createLayer(node['tmp_dir'], cat, ws)
    else:
        # Read CSV tmp file
        with open(tmp_csv_filepath, 'rt') as tmp_file:
            tmp_reader = list(csv.DictReader(tmp_file))

        # Init del layers
        del_layers = []
        for csv_row_tmp in tmp_reader:
            tmp_r = row.Row(csv_row_tmp, config['COLUMNS'])
            del_layers.append(tmp_r.getAttr('layer_name'))

        # Manage layers
        for csv_row in reader:
            r = row.Row(csv_row, config['COLUMNS'])
            add = True
            for csv_row_tmp in tmp_reader:
                tmp_r = row.Row(csv_row_tmp, config['COLUMNS'])
                if r.getAttr('layer_name') == tmp_r.getAttr('layer_name'):
                    # Layer exists in tmp_file: not need to create it
                    add = False
                    del_layers.remove(tmp_r.getAttr('layer_name'))
                    if r.getAttr('date') != config['MAIN']['always_sync'] and r.getAttr('date') != tmp_r.getAttr('date'):
                        # Update layer => data + metadata = create layer with overwrite
                        log.log('Update layer ' + r.getAttr('layer_name') + ': overwrite data and metadata.', 'INFO', 0)
                        r.createLayer(node['tmp_dir'], cat, ws)
                    else:
                        log.log('Layer ' + r.getAttr('layer_name') + ' not modified.', 'INFO', 0)
            if add:
                # layer is in reader but not in tmp_reader
                log.log('Create layer ' + r.getAttr('layer_name') + ' or overwrite it if exists in Geoserver.', 'INFO', 0)
                r.createLayer(node['tmp_dir'], cat, ws)

        for csv_row_del in del_layers:
            # Delete layers of del_layers dictionary
            del_r = row.Row(csv_row_del, config['COLUMNS'])
            log.log('Delete layer ' + del_r.getAttr('layer_name') + '.', 'INFO', 0)
            del_r.deleteLayer(cat)
예제 #9
0
def run():
    # if args.file:
    #     pass
    # else:
    # Get list of nodes files from config node_dir
    nodes_files = [f for f in os.listdir(config['MAIN']['nodes_dir']) if os.path.isfile(os.path.join(config['MAIN']['nodes_dir'], f))]

    count_file = 1
    for nodes_file in nodes_files:
        log.log('', 'INFO', 0)
        log.log(color.OKGREEN + u'*' * 80 + color.ENDC, 'INFO', 0)
        log.log(color.OKGREEN + u'File ' + str(count_file) + '/' + str(len(nodes_files)) + ': ' + nodes_file + '.' + color.ENDC, 'INFO', 0)
        if nodes_file.startswith('_'):
            log.log(u'File skipped.', 'INFO', 0)
        else:
            file_path = os.path.join(config['MAIN']['nodes_dir'], nodes_file)
            with open(file_path, 'r') as json_data:
                data = json.load(json_data)

            log.log(color.BOLD + u'Organisme: ' + data['organisme'] + color.ENDC, 'INFO', 0)

            count_node = 1
            for node in data['nodes']:
                log.log('', 'INFO', 0)
                log.log(color.OKBLUE + u'-' * 80 + color.ENDC, 'INFO', 0)
                log.log(color.OKBLUE + u'Node ' + str(count_node) + '/' + str(len(data['nodes'])) + color.ENDC, 'INFO', 0)
                log.log(u'Description: ' + node['description'], 'INFO', 0)
                if node['active'] == '0':
                    log.log(u'Node disabled.', 'INFO', 0)
                else:
                    # Log node informations
                    log.log(u'Active: ' + node['active'], 'INFO', 0)
                    log.log(u'CSV file: ' + os.path.join(node['src_csv_path'], node['src_csv_file']), 'INFO', 0)
                    log.log(u'XML directory: ' + node['xml_dir'], 'INFO', 0)
                    log.log(u'TMP directory: ' + node['tmp_dir'], 'INFO', 0)

                    # Full path to CSV file
                    csv_filepath = os.path.join(node['tmp_dir'], node['src_csv_file'])
                    # Full path to CSV temp file
                    tmp_csv_file = 'tmp_' + node['src_csv_file']
                    tmp_csv_filepath = os.path.join(node['tmp_dir'], tmp_csv_file)

                    # Create tmp_dir
                    if not os.path.isdir(node['tmp_dir']):
                        os.makedirs(node['tmp_dir'])

                    # Create xml_tmp_dir
                    xml_tmp_dir = os.path.join(node['tmp_dir'], config['MAIN']['xml_tmp_dir'])
                    if not os.path.isdir(xml_tmp_dir):
                        os.makedirs(xml_tmp_dir)

                    log.log(u'Start: ' + str(time.strftime("%Y-%m-%d %H:%M:%S")), 'INFO', 0)
                    # Get file from path or URL
                    helpers.getFile(os.path.join(node['src_csv_path'], node['src_csv_file']), csv_filepath)

                    # Read CSV saved file
                    with open(csv_filepath, 'rt') as csv_file:
                        reader = list(csv.DictReader(csv_file))

                    # metadata_active = 0/1
                    log.log(u'Metadata active: ' + str(config['MAIN']['metadata_active']), 'INFO', 0)
                    if config['MAIN']['metadata_active']:
                        getMetadata(node, reader)

                    log.log(u'Data active: ' + str(config['MAIN']['data_active']), 'INFO', 0)
                    # data_active = 0/1
                    if config['MAIN']['data_active']:
                        getData(node, reader, tmp_csv_filepath)

                    # Copy CSV file to tmp file
                    shutil.copy(csv_filepath, tmp_csv_filepath)

                    log.log(u'End: ' + str(time.strftime("%Y-%m-%d %H:%M:%S")), 'INFO', 0)

                    count_node += 1

            # nodes_file.close()
            count_file += 1
예제 #10
0
 def __init__(self, selenium_driver):
     # type: (object, object, object) -> object
     self.driver = selenium_driver
     self.mylog = log.log()