Exemplos de GrassHdfs em Python

Exemplo n.º 1

Arquivo: hd.hdfs.in.fs.py Projeto: ctu-osgeorel-proj/dp-krejci-2016

def main():
    if options['hdfs'] == '@grass_data_hdfs':
        LOCATION_NAME = grass.gisenv()['LOCATION_NAME']
        MAPSET = grass.gisenv()['MAPSET']
        MAPSET_PATH = os.path.join('grass_data_hdfs', LOCATION_NAME, MAPSET, 'external')
        options['hdfs'] = MAPSET_PATH

    if options['local']:
        transf = GrassHdfs(options['driver'])
        transf.upload(options['local'], options['hdfs'])

Exemplo n.º 2

Arquivo: hd.hdfs.in.fs.py Projeto: lucadelu/grass-addons

def main():
    if options["hdfs"] == "@grass_data_hdfs":
        LOCATION_NAME = grass.gisenv()["LOCATION_NAME"]
        MAPSET = grass.gisenv()["MAPSET"]
        MAPSET_PATH = os.path.join("grass_data_hdfs", LOCATION_NAME, MAPSET,
                                   "external")
        options["hdfs"] = MAPSET_PATH

    if options["local"]:
        transf = GrassHdfs(options["driver"])
        transf.upload(options["local"], options["hdfs"])

Exemplo n.º 3

Arquivo: hd.hdfs.in.fs.py Projeto: ctu-geoforall-lab-projects/dp-krejci-2016

def main():
    if options['hdfs'] == '@grass_data_hdfs':
        LOCATION_NAME = grass.gisenv()['LOCATION_NAME']
        MAPSET = grass.gisenv()['MAPSET']
        MAPSET_PATH = os.path.join('grass_data_hdfs', LOCATION_NAME, MAPSET,
                                   'external')
        options['hdfs'] = MAPSET_PATH

    if options['local']:
        transf = GrassHdfs(options['driver'])
        transf.upload(options['local'], options['hdfs'])

Exemplo n.º 4

Arquivo: hd.hdfs.in.vector.py Projeto: ctu-osgeorel-proj/dp-krejci-2016

def main():
    transf = GrassHdfs(options['driver'])
    if options['hdfs'] == '@grass_data_hdfs':
        options['hdfs'] = transf.get_path_grass_dataset()

    grass.message(options['hdfs'])
    grass_map = {"map": options['map'],
                 "layer": options['layer'],
                 "type": options['type'],
                 }

    json = JSONBuilder(grass_map)
    json = json.get_JSON()

    grass.message('upload %s' % json)

    transf.upload(json, options['hdfs'])

Exemplo n.º 5

Arquivo: hd.hdfs.in.vector.py Projeto: lucadelu/grass-addons

def main():
    transf = GrassHdfs(options["driver"])
    if options["hdfs"] == "@grass_data_hdfs":
        options["hdfs"] = transf.get_path_grass_dataset()

    grass.message(options["hdfs"])
    grass_map = {
        "map": options["map"],
        "layer": options["layer"],
        "type": options["type"],
    }

    json = JSONBuilder(grass_map)
    json = json.get_JSON()

    grass.message("upload %s" % json)

    transf.upload(json, options["hdfs"])

Exemplo n.º 6

Arquivo: hd.hdfs.in.vector.py Projeto: gracedoherty/grass-addons

def main():
    transf = GrassHdfs(options['driver'])
    if options['hdfs'] == '@grass_data_hdfs':
        options['hdfs'] = transf.get_path_grass_dataset()

    grass.message(options['hdfs'])
    grass_map = {
        "map": options['map'],
        "layer": options['layer'],
        "type": options['type'],
    }

    json = JSONBuilder(grass_map)
    json = json.get_JSON()

    grass.message('upload %s' % json)

    transf.upload(json, options['hdfs'])

Exemplo n.º 7

Arquivo: hd.hdfs.out.vector.py Projeto: ctu-osgeorel-proj/dp-krejci-2016

def main():
    tmp_dir = os.path.join(get_tmp_folder(), options['out'])

    if os.path.exists(tmp_dir):
        shutil.rmtree(tmp_dir)

    transf = GrassHdfs(options['driver'])
    table_path = options['hdfs']

    if options['table']:
        conn = ConnectionManager()
        conn.get_current_connection('hiveserver2')

        if not conn.get_current_connection('hiveserver2'):
            grass.fatal("Cannot connet to hive for table description. "
                        "Use param hdfs without param table")

        hive = conn.get_hook()
        table_path = hive.find_table_location(options['table'])
        tmp_dir = os.path.join(tmp_dir,options['table'])


    if not transf.download(hdfs=table_path,
                           fs=tmp_dir):
        return


    files = os.listdir(tmp_dir)
    map_string=''
    for block in files:
        map='%s_%s'%(options['out'],block)
        block=os.path.join(tmp_dir,block)

        map_build = GrassMapBuilderEsriToEsri(block,
                                              map,
                                              options['attributes'])
        try:
            map_build.build()
            map_string+='%s,'%map
        except Exception ,e:
            grass.warning("Error: %s\n     Map < %s >  conversion failed"%(e,block))

Exemplo n.º 8

Arquivo: hd.hdfs.out.vector.py Projeto: wiesnerfriedman/grass-addons

def main():
    tmp_dir = os.path.join(get_tmp_folder(), options['out'])

    if os.path.exists(tmp_dir):
        shutil.rmtree(tmp_dir)

    transf = GrassHdfs(options['driver'])
    table_path = options['hdfs']

    if options['table']:
        conn = ConnectionManager()
        conn.get_current_connection('hiveserver2')

        if not conn.get_current_connection('hiveserver2'):
            grass.fatal("Cannot connet to hive for table description. "
                        "Use param hdfs without param table")

        hive = conn.get_hook()
        table_path = hive.find_table_location(options['table'])
        tmp_dir = os.path.join(tmp_dir,options['table'])


    if not transf.download(hdfs=table_path,
                           fs=tmp_dir):
        return


    files = os.listdir(tmp_dir)
    map_string = ''
    for block in files:
        map = '%s_%s' % (options['out'],block)
        block = os.path.join(tmp_dir,block)

        map_build = GrassMapBuilderEsriToEsri(block,
                                              map,
                                              options['attributes'])
        try:
            map_build.build()
            map_string += '%s,' % map
        except Exception,e:
            grass.warning("Error: %s\n     Map < %s >  conversion failed" % (e,block))