Example #1
0
    def import_file(username,
                    repo_base,
                    repo,
                    table,
                    file_name,
                    file_format='CSV',
                    delimiter=',',
                    header=True,
                    encoding='ISO-8859-1',
                    quote_character='"'):
        # check for permissions
        delimiter = delimiter.decode('string_escape')

        res = DataHubManager.has_repo_privilege(username, repo_base, repo,
                                                'CREATE')
        if not res:
            raise PermissionDenied(
                'Access denied. Missing required privileges.')

        # prepare some variables
        file_path = user_data_path(repo_base, repo, file_name)
        table_name, _ = os.path.splitext(file_name)
        table_name = clean_str(table_name, 'table')
        dh_table_name = '%s.%s.%s' % (repo_base, repo, table_name)

        # open the file
        f = codecs.open(file_path, 'r', 'ISO-8859-1')
        data = csv.reader(f, delimiter=delimiter)

        # create a table for the data
        cells = data.next()
        columns = [clean_str(str(i), 'col') for i in range(0, len(cells))]
        if header:
            columns = map(lambda x: clean_str(x, 'col'), cells)
        columns = rename_duplicates(columns)

        query = 'CREATE TABLE %s (%s text' % (dh_table_name, columns[0])
        for column in columns[1:len(columns)]:
            query += ', %s %s' % (column, 'text')
        query += ')'

        manager = DataHubManager(user=username, repo_base=repo_base)
        manager.execute_sql(query=query)

        # populate the newly created table with data from the csv
        superuser_con = DataHubConnection(
            user=settings.DATABASES['default']['USER'],
            password=settings.DATABASES['default']['USER'],
            repo_base=repo_base)
        return superuser_con.import_file(table_name=dh_table_name,
                                         file_path=file_path,
                                         file_format=file_format,
                                         delimiter=delimiter,
                                         header=header,
                                         encoding=encoding,
                                         quote_character=quote_character)
Example #2
0
    def import_file(username, repo_base, repo, table, file_name,
                    file_format='CSV', delimiter=',', header=True,
                    encoding='ISO-8859-1', quote_character='"'):
        # check for permissions
        delimiter = delimiter.decode('string_escape')

        res = DataHubManager.has_repo_privilege(
            username, repo_base, repo, 'CREATE')
        if not res:
            raise PermissionDenied(
                'Access denied. Missing required privileges.')

        # prepare some variables
        file_path = user_data_path(repo_base, repo, file_name)
        table_name, _ = os.path.splitext(file_name)
        table_name = clean_str(table_name, 'table')
        dh_table_name = '%s.%s.%s' % (repo_base, repo, table_name)

        # open the file
        f = codecs.open(file_path, 'r', 'ISO-8859-1')
        data = csv.reader(f, delimiter=delimiter)

        # create a table for the data
        cells = data.next()
        columns = [clean_str(str(i), 'col') for i in range(0, len(cells))]
        if header:
            columns = map(lambda x: clean_str(x, 'col'), cells)
        columns = rename_duplicates(columns)

        query = 'CREATE TABLE %s (%s text' % (dh_table_name, columns[0])
        for column in columns[1:len(columns)]:
            query += ', %s %s' % (column, 'text')
        query += ')'

        manager = DataHubManager(user=username, repo_base=repo_base)
        manager.execute_sql(query=query)

        # populate the newly created table with data from the csv
        superuser_con = DataHubConnection(
            user=settings.DATABASES['default']['USER'],
            password=settings.DATABASES['default']['USER'],
            repo_base=repo_base)
        return superuser_con.import_file(
            table_name=dh_table_name,
            file_path=file_path,
            file_format=file_format,
            delimiter=delimiter,
            header=header,
            encoding=encoding,
            quote_character=quote_character)
Example #3
0
 def import_file(repo_base, table_name, file_path, file_format='CSV',
     delimiter=',', header=True, encoding='ISO-8859-1', quote_character='"'):
   superuser_con = DataHubConnection(
       user=settings.DATABASES['default']['USER'],
       password=settings.DATABASES['default']['USER'],
       repo_base=repo_base)
   return superuser_con.import_file(
       table_name=table_name,
       file_path=file_path,
       file_format=file_format,
       delimiter=delimiter,
       header=header,
       encoding=encoding,
       quote_character=quote_character)
Example #4
0
 def import_file(repo_base,
                 table_name,
                 file_path,
                 file_format='CSV',
                 delimiter=',',
                 header=True,
                 encoding='ISO-8859-1',
                 quote_character='"'):
     superuser_con = DataHubConnection(
         user=settings.DATABASES['default']['USER'],
         password=settings.DATABASES['default']['USER'],
         repo_base=repo_base)
     return superuser_con.import_file(table_name=table_name,
                                      file_path=file_path,
                                      file_format=file_format,
                                      delimiter=delimiter,
                                      header=header,
                                      encoding=encoding,
                                      quote_character=quote_character)
Example #5
0
 def import_file(
     repo_base,
     table_name,
     file_path,
     file_format="CSV",
     delimiter=",",
     header=True,
     encoding="ISO-8859-1",
     quote_character='"',
 ):
     superuser_con = DataHubConnection(
         user=settings.DATABASES["default"]["USER"],
         password=settings.DATABASES["default"]["USER"],
         repo_base=repo_base,
     )
     return superuser_con.import_file(
         table_name=table_name,
         file_path=file_path,
         file_format=file_format,
         delimiter=delimiter,
         header=header,
         encoding=encoding,
         quote_character=quote_character,
     )