Example #1
0
 def exportJsonDB(json_data, frameNum):
     """ Exports the JSON data to the Accumulo database """
     conn = Accumulo(host="localhost",
                     port=50096,
                     user="******",
                     password="******")
     json_data_parsed = json.loads(
         json_data)  #put json data back into dictionary
     table = json_data_parsed['videoMetadata'][
         'videoName']  #get the video name and set that as the table name
     table = table.replace('.', '_')
     table = table.encode('ascii', 'ignore')
     if not conn.table_exists(table):
         conn.create_table(table)
     m = Mutation("row_%d" %
                  frameNum)  #table row number is the frame number
     m.put(cf="cf2", cq="cq2", val=json_data_parsed['imageBase64']
           )  #saves the frame image separately from the metadata
     if 'LabeledImage' in json_data_parsed.keys():
         m.put(cf="cf3", cq="cq3", val=json_data_parsed['LabeledImage']
               )  #saves the labeled image separately from the metadata
         json_data_parsed.pop(
             'LabeledImage',
             None)  #delete the base64 representation of the labeled frame
     json_data_parsed.pop(
         'imageBase64',
         None)  #delete the base64 representation of the frame
     json_data = json.dumps(json_data_parsed)
     m.put(cf="cf1", cq="cq1",
           val=json_data)  #set the first column to now only the metadata.
     conn.write(table, m)
     conn.close()
 def _connect(self, host, port, user, password):
     try:
         self.__connection = Accumulo(host, port, user, password)
         self.__log.debug('Connected to StaticFile Store')
     except Exception as e:
         self.__log.exception('Error while connecting to StaticFile Store: %s' % str(e))
         raise Exception('Error while connecting to StaticFile Store: %s' % str(e))
Example #3
0
 def printTableDB(table):
     """ Displays the data in the database """
     conn = Accumulo(host="localhost",
                     port=50096,
                     user="******",
                     password="******")
     for entry in conn.scan(table):
         print(entry.row, entry.cf, entry.cq, entry.cv, entry.ts, entry.val)
     conn.close()
Example #4
0
    def new(cls,
            elems,
            lbound,
            rbound,
            coin=BaseCoin(),
            conn_info=ConnInfo('localhost', 42424, 'root', 'secret'),
            table='__ADS_metadata___',
            elemclass=IntElem):
        """ Create a new skiplist that stores all of its data inside an
            Accumulo instance.

            Arguments:

            cls - the class implementing this class method
            elems - the elements to create the skiplist over
            lbound, rbound - the left and right boundary elements of the list
            coin - the source of randomness to use
                   (see pace.ads.skiplist.coin)
            conn_info - how to connect to the Accumulo instance being used
            table - the name of the table to store the ADS in
            elemclass - the class to use to store the elements in the skiplist
        """

        sl = cls(None, lbound, rbound, coin)

        if conn_info is not None:
            # For connecting to a live Accumulo instance
            host, port, user, password = conn_info
            conn = Accumulo(host=conn_info.host,
                            port=conn_info.port,
                            user=conn_info.user,
                            password=conn_info.password)
        else:
            # For testing/debug
            conn = FakeConnection()

        sl.conn = conn
        sl.table = table
        sl.elemclass = elemclass

        if not conn.table_exists(table):
            conn.create_table(table)

        right = cls.nodeclass.newnode(sl, None, None, rbound, True)
        left = cls.nodeclass.newnode(sl, None, right, lbound, True)

        sl.root = left

        for elem in elems:
            sl.insert(elem)

        return sl
 def __init__(self,
              host="localhost",
              port=42424,
              user="******",
              password="******",
              num_trials=100,
              filename='default_file.txt',
              seed=None,
              signer_ids=test_ids,
              pki=test_pki):
     self.conn = Accumulo(host=host,
                          port=port,
                          user=user,
                          password=password)
     self.num_trials = num_trials
     self.filename = filename
     self.seed = seed
     self.signer_ids = signer_ids
     self.pki = pki
Example #6
0
"""
Reads lines from a data file and inserts a number of records
"""

from pyaccumulo import Accumulo, Mutation, Range
import settings
import sys
sys.path
sys.path.append('/bdsetup')

table = "well_logs"
table1 = "drill_logs"
conn = Accumulo(host=settings.HOST,
                port=settings.PORT,
                user=settings.USER,
                password=settings.PASSWORD)

if conn.table_exists(table):
    conn.delete_table(table)

conn.create_table(table)
wr = conn.create_batch_writer(table)

print "Ingesting some data ..."
f = open("/bdsetup/acculog.txt", "rb")
for i in range(250):
    line = f.readline().rstrip()
    label = '%04d' % i
    mut = Mutation('r_%s' % label)
    mut.put(cq='cq1', val=line)
    #mut.put(cf='cf_%s'%label, cq='cq1', val=line)
Example #7
0
def main():
    parser = OptionParser()
    parser.add_option("-v",
                      '--verbose',
                      dest="verbose",
                      action="store_true",
                      default=False,
                      help="Verbose output")
    accumulo_group = OptionGroup(
        parser, 'Options that control the accumulo connection')
    accumulo_group.add_option('--host',
                              dest='host',
                              default='localhost',
                              help='Host for Accumulo. Default: localhost')
    accumulo_group.add_option('--user',
                              dest='user',
                              default='root',
                              help='User for Accumulo. Default: root')
    accumulo_group.add_option('--password',
                              dest='password',
                              default='secret',
                              help='Password for Accumulo user. Default: ...')
    accumulo_group.add_option('--port',
                              dest='port',
                              type='int',
                              default=42424,
                              help="Port for Accumulo. Default: 42424")
    parser.add_option_group(accumulo_group)

    output_group = OptionGroup(parser, 'Options that control output')
    output_group.add_option('--log-file',
                            dest='log_file',
                            default='output.log',
                            help='Output file for performance numbers')
    output_group.add_option('--table-prefix',
                            dest='table_prefix',
                            default='perf',
                            help='Prefix used for data tables')
    output_group.add_option('--profile',
                            dest='profile',
                            action='store_true',
                            default=False,
                            help="Profiles encryption code")
    output_group.add_option(
        '--cache_key',
        dest='cache_key',
        action='store_true',
        default=False,
        help='Keys are now cached during encryption and decryption')
    output_group.add_option(
        '--use_accumulo_keystore',
        dest='accumulo_keystore',
        action='store_true',
        default=False,
        help=
        "Keys are stored in Accumulo if option is included, otherwise they are stored locally"
    )
    parser.add_option_group(output_group)

    test_group = OptionGroup(parser,
                             "Options that control what tests are being run")
    test_group.add_option('--all',
                          dest='all',
                          action='store_true',
                          default=False,
                          help='Runs all the different tests')
    test_group.add_option(
        '--non-ceabac',
        dest='non_ceabac',
        action='store_true',
        default=False,
        help='Runs the non-CEABAC tests with a simple schema')
    test_group.add_option('--ceabac',
                          dest='ceabac',
                          action='store_true',
                          default=False,
                          help='Runs the CEABAC tests with a simple schema')
    test_group.add_option(
        '--vis-ceabac',
        dest='vis_ceabac',
        action='store_true',
        default=False,
        help='Runs CEABAC in CBC mode with varying visibility fields')
    test_group.add_option('--diff_schemas_ceabac',
                          dest='diff_ceabac',
                          action='store_true',
                          default=False,
                          help='Runs several different schemas for VIS_CBC')
    test_group.add_option('--diff_schemas_non_ceabac',
                          dest='diff_non_ceabac',
                          action='store_true',
                          default=False,
                          help='Runs several different schemas for AES_CBC')

    test_group.add_option(
        '--mixed_schemas',
        dest='mixed_schemas',
        action='store_true',
        default=False,
        help='Runs a set of schemas where the schemes are both CEABAC and not')
    parser.add_option_group(test_group)

    entries_group = OptionGroup(
        parser, "Options that control how many entries are run")
    entries_group.add_option('--num_entries',
                             dest='num_entries',
                             type='int',
                             default=1000,
                             help='Total number of cells being run')
    entries_group.add_option('--num_rows',
                             dest='num_rows',
                             type='int',
                             default=100,
                             help='Total number of rows being run')
    parser.add_option_group(entries_group)

    (cl_flags, _) = parser.parse_args()

    #set up logging
    if cl_flags.verbose:
        log_level = logging.DEBUG
    else:
        log_level = logging.INFO
    logging.basicConfig(filename=cl_flags.log_file,
                        level=log_level,
                        format='%(levelname)s-%(asctime)s: %(message)s')

    logger = logging.getLogger("performance_testing")

    #check inputs
    if cl_flags.all and (cl_flags.non_ceabac or cl_flags.ceabac
                         or cl_flags.vis_ceabac):
        logger.error(
            '--all is already specified, do not need to define other tests to run'
        )

    #create accumulo connection
    conn = Accumulo(host=cl_flags.host,
                    port=cl_flags.port,
                    user=cl_flags.user,
                    password=cl_flags.password)

    #create benchmarker
    if cl_flags.cache_key:
        logger.info('Using the caching version of the pki')
        pki = DummyCachingEncryptionPKI(
            conn=conn if cl_flags.accumulo_keystore else None)
    else:
        pki = DummyEncryptionPKI(
            conn=conn if cl_flags.accumulo_keystore else None)

    benchmarker = Benchmarker(logger=logger, pki=pki, conn=conn)

    if cl_flags.all:
        run_non_ceabac(benchmarker, cl_flags.table_prefix, logger,
                       cl_flags.profile, cl_flags)
        run_ceabac(benchmarker, cl_flags.table_prefix, logger,
                   cl_flags.profile, cl_flags)
        run_vis_ceabac(benchmarker, cl_flags.table_prefix, logger,
                       cl_flags.profile, cl_flags)
        run_diff_ceabac(benchmarker, cl_flags.table_prefix, logger,
                        cl_flags.profile, cl_flags)
        run_diff_non_ceabac(benchmarker, cl_flags.table_prefix, logger,
                            cl_flags.profile, cl_flags)
        run_mixed_schemas(benchmarker, cl_flags.table_prefix, logger,
                          cl_flags.profile, cl_flags)

    if cl_flags.non_ceabac:
        run_non_ceabac(benchmarker, cl_flags.table_prefix, logger,
                       cl_flags.profile, cl_flags)

    if cl_flags.ceabac:
        run_ceabac(benchmarker, cl_flags.table_prefix, logger,
                   cl_flags.profile, cl_flags)

    if cl_flags.vis_ceabac:
        run_vis_ceabac(benchmarker, cl_flags.table_prefix, logger,
                       cl_flags.profile, cl_flags)

    if cl_flags.diff_ceabac:
        run_diff_ceabac(benchmarker, cl_flags.table_prefix, logger,
                        cl_flags.profile, cl_flags)

    if cl_flags.diff_non_ceabac:
        run_diff_non_ceabac(benchmarker, cl_flags.table_prefix, logger,
                            cl_flags.profile, cl_flags)

    if cl_flags.mixed_schemas:
        run_mixed_schemas(benchmarker, cl_flags.table_prefix, logger,
                          cl_flags.profile, cl_flags)