def get_bug_report(data_prefix, vectorized_data, bug_report_id):
    bug_report_index_collection = UnQLite(
        data_prefix + "_bug_report_index_collection_index_db")
    bug_report = pickle.loads(bug_report_index_collection[bug_report_id])
    bug_report_index_collection.close()
    index = bug_report['report']
    return vectorized_data[index, :]
Exemplo n.º 2
0
 def __init__(self, db_path=''):
     self.db_path = db_path
     self.store = UnQLite(
     )  #loading db from databases freezes the process!!
     self.hits = 0
     self.misses = 0
     self.uncommited_recs = 0
Exemplo n.º 3
0
class UniqLiteFactory:
    def __init__(self):
        self.conn = None

    def connect(self):
        self.conn = UnQLite(filename)
        return self.conn

    def disconnect(self):
        self.conn.close()

    def create_collection(self, name):
        db = self.connect()
        collection = db.collection(name)
        collection.create()
        print "collection " + name + " created successfully "
        self.disconnect()

    def drop_collection(self, name):
        db = self.connect()
        collection = db.collection(name)
        collection.drop()
        print "collection " + name + " droped successfully "
        self.disconnect()

    def insert(self, collection_name, data=None):
        db = self.connect()
        collection_name = db.collection(collection_name)
        collection_name.store(data)
        self.disconnect()
Exemplo n.º 4
0
 def getTimesFromDB(self, dbPath):
     db = UnQLite(DB_PATH)
     times = db.collection("times")
     if not times.exists():
         raise Exception("No data to retrieve")
     self.dbData = times.all()
     return self
Exemplo n.º 5
0
class Compilation_error_db:
    def __init__(self, db_path=''):
        self.db_path = db_path
        self.store = UnQLite()  #loading db from databases freezes the process!!
        self.hits = 0
        self.misses = 0
        self.uncommited_recs = 0

    # keeping prog_id for backward compatibility
    def get_errors(self, prog_id, prog):
        #print("get_errors here")
        if prog in self.store:
            err_msg = self.store[prog]
            errs = get_error_list(err_msg)
            self.hits += 1
        else:
            errs, err_msg = compilation_errors(prog)
            self.store[prog] = err_msg
            self.misses += 1
            self.uncommited_recs += 1

            if self.uncommited_recs > 0 and self.uncommited_recs % 250 == 0:
                self.commit()
        return errs

    def close(self):
        self.store.close()

    def commit(self, ):
        cnt = self.uncommited_recs
        self.uncommited_recs = 0
        self.store.commit()

    def __len__(self):
        return len(self.store)
Exemplo n.º 6
0
 def consolidateToDB(self):
     db = UnQLite(DB_PATH)
     times = db.collection("times")
     if not times.exists():
         times.create()
     for t in self.timeMap.keys():
             times.store({t: self.timeMap[t]})
     db.commit()
Exemplo n.º 7
0
    def setUp(self):
        self.app = bottle.Bottle(catchall=False)
        _, filename = tempfile.mkstemp(suffix='.unqlite')
        self.plugin = self.app.install(unqlite.Plugin(filename=filename))

        self.conn = UnQLite(filename)
        self.conn.collection('todo').create()
        self.conn.close()
Exemplo n.º 8
0
 def db(cls):
     if cls._database is not None:
         return cls._database
     else:
         if not bool(os.environ.get(_UNQLITE_READ_ONLY_ENV, False)):
             cls._database = UnQLite(cls.database_file)
         else:
             cls._database = UnQLite(cls.database_file, flags=0x00000001)
         atexit.register(cls._database.close)
         return cls._database
Exemplo n.º 9
0
    def __init__(self, filename=None):
        assert filename is not None
        if not os.path.exists(filename):
            directory = Path(filename[:filename.rfind('/')])
            directory.mkdir(parents=True, exist_ok=True)

        self.__db = UnQLite(filename)

        if self.__db_get('master_password_hash'):
            current_app.config['INIT_STATE'] = 2
Exemplo n.º 10
0
def insert_unqlite_items(number):
    db = UnQLite('tmp.unqlite')
    items = db.collection('items')
    items.create()
    for x in xrange(number):
        items.store([{
            'a': str(x),
            'b': '2',
            'c': '3',
        }])
Exemplo n.º 11
0
 def __init__(self, location=None):
     self.location = location
     if type(self.location) == str and len(self.location) > 0:
         logger.debug("Connecting to database at {}".format(
             os.path.abspath(location)))
         self.db = UnQLite(self.location)
     else:
         # in-memory database
         logger.debug("Creating an in-memory database.")
         self.db = UnQLite()
     self.collections = dict()
Exemplo n.º 12
0
 def get_db(source='orders', suffix='', directory=''):
     col_path = source + suffix
     base_path = directory
     if not base_path.endswith('/'):
         base_path = base_path + '/'
     if not os.path.isdir(base_path):
         os.mkdir(base_path)
     db = UnQLite(base_path + col_path +
                  '.db')  # Create an in-memory database.
     col = db.collection(col_path)
     col.create()
     return col, db
def extract_enriched_api(data_prefix, bug_report_full_sha):
    data = sparse.load_npz(data_prefix + '_raw_count_data.npz')
    bug_report_files_collection_db = UnQLite(data_prefix +
                                             "_bug_report_files_collection_db",
                                             flags=0x00000100 | 0x00000001)

    current_files = pickle.loads(
        bug_report_files_collection_db[bug_report_full_sha])
    bug_report_files_collection_db.close()

    bug_report_id = bug_report_full_sha[0:7]

    shas = current_files['shas']
    class_name_lookup = current_files['class_name_to_sha']

    bug_report_data = []
    bug_report_lookup = {}

    n_rows = 0

    for ast_sha in shas:
        ast_data, lookup, current_ast_sha = add_types_source_to_bug_report_data(
            data, data_prefix, class_name_lookup, ast_sha)
        current_index = n_rows
        bug_report_data.append(ast_data)
        for k in lookup:
            lookup[k] += current_index
        bug_report_lookup[current_ast_sha] = lookup
        n_rows += ast_data.shape[0]

    bug_report_row = get_bug_report(data_prefix, data, bug_report_id)
    bug_report_data.append(bug_report_row)

    bug_report_data_matrix = sparse.vstack(bug_report_data)

    sparse.save_npz(
        data_prefix + '_' + bug_report_id + '_partial_enriched_api',
        bug_report_data_matrix)
    with open(
            data_prefix + '_' + bug_report_id +
            '_partial_enriched_api_index_lookup', 'w') as outfile:
        json.dump(bug_report_lookup, outfile)

    transformer = TfidfTransformer()
    tf_idf_data = transformer.fit_transform(bug_report_data_matrix)
    sparse.save_npz(data_prefix + '_' + bug_report_id + '_tfidf_enriched_api',
                    tf_idf_data)

    #    print "bug_report_id", bug_report_id

    return bug_report_id
Exemplo n.º 14
0
 def __init__(self, path, db_path, options=dict()):
     self.board = Board()
     self.db = UnQLite(db_path)
     self.engine = Popen(path,
                         universal_newlines=True,
                         stdin=PIPE,
                         stdout=PIPE)
     self._put('uci')
     self._ready()
     for option, val in options.items():
         self._set_option(option, val)
     self.num_games = 1
     while True:
         self.board.reset()
         self.learn(200)
Exemplo n.º 15
0
def get_user(user_uuid):
    db = UnQLite(db_file)
    user_col = get_collection('user')
    try:
        return user_col.fetch(db[user_uuid])
    except KeyError:
        return None
Exemplo n.º 16
0
def create_demo_db(db_file, data_path, user_count, truncate=True):
    if os.path.isfile(db_file):
        if truncate:
            os.remove(db_file)
    db = UnQLite(db_file)
    create_users(db, data_path, user_count)
    create_tags(db, data_path)
Exemplo n.º 17
0
 def __init__(
     self,
     *,
     mydomains: List[str],
     local_delivery_handler: LocalDeliveryHandler,
     database: UnQLite,
     smtpd_auth_handler: SMTPAuthHandler,
     hostname: str,
     self_name: str = "mailboat.transfer_agent",
     smtpd_port: int = 8025,
     custom_queue: Optional[EmailQueue] = None,
 ) -> None:
     self.mydomains = mydomains
     self.database = database
     self.name = self_name
     self.hostname = hostname
     self.queue = (custom_queue
                   if custom_queue else UnQLiteEmailMessageQueue(
                       database.collection("{}.queue".format(self_name))))
     self.delivery_tasks: List[DeliveryTask] = []
     self.smtpd_controller = Controller(
         _SMTPDHandler(
             self.handle_message,
             smtp_auth_handler=smtpd_auth_handler,
         ),
         port=smtpd_port,
         hostname=hostname,
     )
     self.local_delivery_handler = local_delivery_handler
     self._task_deliveryman = asyncio.ensure_future(
         self._cothread_deliveryman())
Exemplo n.º 18
0
def dump_info(dirinfo, out_path):
    """Dump FileInfo into a UnQLite database"""
    out_path.unlink(missing_ok=True)
    with UnQLite(out_path.as_posix()) as db:
        with db.transaction():
            for info in dirinfo:
                db[info.name] = json.dumps(info.__dict__)
 def __getitem__(self, entity) -> UnQLite:
     """
     Get UnQLite index by entity name
     :param entity: 
     :return: 
     """
     return UnQLite(self.uri_index_files_by_entity[entity],
                    flags=UNQLITE_OPEN_READONLY)
Exemplo n.º 20
0
 def __init__(self, filepath):
     self.filepath = filepath
     self.name = os.path.basename(filepath)
     # In memory JSON store
     self.db = UnQLite()
     self.workbook = self.load_from_file(filepath)
     self.data = self.extract_as_python()
     self.to_db()
Exemplo n.º 21
0
def get_unqlite(name="default", path=None):
    '''Get unqlite object
    
    :param name: unqlite db name, will use the name to get path from settings.UNQLITE_DB_NAMES
    :param path: unqlite db file path, if None, will use configuration in get from name
    '''

    from uliweb import settings
    from unqlite import UnQLite
    if not path:
        path = settings.UNQLITE_DB_NAMES.get(name)
    if path == ':mem:':
        #always use a same unqlite of ':mem:'
        if not _v.mem_unqlite:
            _v.mem_unqlite = UnQLite(path)
        return _v.mem_unqlite
    return UnQLite(path)
Exemplo n.º 22
0
def write():
    db = UnQLite(DB_FILE)

    with db.transaction():
        db.collection('users').create()

    for line in FIRST_LINES:
        with db.transaction():
            stories = db.collection('stories')
            stories.create()

            # Are there stories that have the same first line?
            same_first_line = stories.filter(
                lambda story: story['lines'][0].get('text') == line)
            if same_first_line:
                continue

            stories.store([{
                "max_lines": MAX_LINES,
                "locked": False,
                "locked_by": None,
                "locked_at": None,
                "lines": [{
                    "text": line
                }]
            }])
Exemplo n.º 23
0
    def __init__(self, location=None):

        try:
            from unqlite import UnQLite
        except ImportError:
            raise ImportError(
                "The unqlite library is required for this feature.")

        self.location = location
        if type(self.location) == str and len(self.location) > 0:
            logger.debug("Connecting to database at {}".format(
                os.path.abspath(location)))
            self.db = UnQLite(self.location)
        else:
            # in-memory database
            logger.debug("Creating an in-memory database.")
            self.db = UnQLite()
        self.collections = dict()
Exemplo n.º 24
0
def main():
    print("Start", datetime.datetime.now().isoformat())
    before = default_timer()
    bug_report_file_path = sys.argv[1]
    print("bug report file path", bug_report_file_path)
    data_prefix = sys.argv[2]
    print("data prefix", data_prefix)

    bug_reports = load_bug_reports(bug_report_file_path)

    ast_cache_db = UnQLite(data_prefix + "_ast_cache_collection_db")

    vectorize(ast_cache_db, bug_reports, data_prefix)
    after = default_timer()
    total = after - before
    print("End", datetime.datetime.now().isoformat())
    print("total time ", total)
    ast_cache_db.close()
Exemplo n.º 25
0
    async def test_performance_local_delivery_in_pure_memory_queue(
            self, unused_tcp_port: int):
        TEST_MAIL_NUMBER = 100
        virtual_box = []
        database = UnQLite(":mem:")

        async def delivery_handler(email: EmailMessage):
            virtual_box.append(email)

        ta = TransferAgent(
            mydomains=["localhost"],
            hostname="localhost",
            database=database,
            local_delivery_handler=delivery_handler,
            smtpd_auth_handler=smtpd_auth_rejectall,
            custom_queue=MemoryEmailQueue(),
            smtpd_port=unused_tcp_port,
        )

        try:
            ta.start()
            email = EmailMessage()
            email["Message-Id"] = "<test1@localhost>"
            email["To"] = "user@localhost"
            email["From"] = "qa@localhost"
            t1 = perf_counter()
            lost_mail_count = 0
            for x in range(0, TEST_MAIL_NUMBER):
                try:
                    await aiosmtplib.send(email,
                                          hostname="localhost",
                                          port=unused_tcp_port)
                except:
                    lost_mail_count += 1

            async def wait_virtual_box():
                while len(virtual_box) < TEST_MAIL_NUMBER:
                    await asyncio.sleep(0)

            await asyncio.wait_for(wait_virtual_box(), 12)
            t2 = perf_counter()
            result = t2 - t1
            logging.warning(
                "MemoryEmailQueue: %f sec./%smails, lost=%d",
                result,
                TEST_MAIL_NUMBER,
                lost_mail_count,
            )
            assert result < (
                TEST_MAIL_NUMBER / 100 *
                4), "the MTA performance should be 25 per second at least"
        finally:
            ta.destory()
            if len(virtual_box) != TEST_MAIL_NUMBER:
                pytest.fail("except {} mails, got {}".format(
                    TEST_MAIL_NUMBER, len(virtual_box)))
Exemplo n.º 26
0
class TestTransaction(BaseTestCase):
    """
    We must use a file-based database to test the transaction functions. See
    http://unqlite.org/forum/trouble-with-transactions+1 for details.
    """

    def setUp(self):
        self._filename = "test.db"
        self.db = UnQLite(self._filename)

    def tearDown(self):
        try:
            self.db.close()
        except:
            pass
        if os.path.exists(self._filename):
            os.unlink(self._filename)

    def test_transaction(self):
        @self.db.commit_on_success
        def _test_success(key, value):
            self.db[key] = value

        @self.db.commit_on_success
        def _test_failure(key, value):
            self.db[key] = value
            raise Exception("intentional exception raised")

        _test_success("k1", "v1")
        self.assertEqual(self.db["k1"], "v1")

        self.assertRaises(Exception, lambda: _test_failure("k2", "v2"))
        self.assertRaises(KeyError, lambda: self.db["k2"])

    def test_explicit_transaction(self):
        self.db.close()
        self.db.open()
        self.db.begin()
        self.db["k1"] = "v1"
        self.db.rollback()

        self.assertRaises(KeyError, lambda: self.db["k1"])
Exemplo n.º 27
0
class BaseTestCase(unittest.TestCase):
    def setUp(self):
        super(BaseTestCase, self).setUp()
        self.db = UnQLite(':mem:')
        self._filename = 'test.db'
        self.file_db = UnQLite(self._filename)

    def tearDown(self):
        try:
            self.file_db.close()
        except:
            pass
        if os.path.exists(self._filename):
            os.unlink(self._filename)

    def store_range(self, n, db=None):
        if db is None:
            db = self.db
        for i in range(n):
            db['k%s' % i] = str(i)
Exemplo n.º 28
0
class BaseTestCase(unittest.TestCase):
    def setUp(self):
        super(BaseTestCase, self).setUp()
        self.db = UnQLite(':mem:')
        self._filename = 'test.db'
        self.file_db = UnQLite(self._filename)

    def tearDown(self):
        try:
            self.file_db.close()
        except:
            pass
        if os.path.exists(self._filename):
            os.unlink(self._filename)

    def store_range(self, n, db=None):
        if db is None:
            db = self.db
        for i in range(n):
            db['k%s' % i] = str(i)
Exemplo n.º 29
0
	def __init__(self, brpath, dbpath):
		self.br = Engine(brpath)
		# f*****g cute chess
		self.br.write('setoption name Threads value 3')
		self.br.write('setoption name Hash value 4096')
		# f*****g cute chess
		self.db = UnQLite(dbpath)
		self.board = Board()
		self.func_book = {
			'uci': self.uci,
			'isready': self.isready,
			'go': self.go,
			'quit': self.quit,
			'position': self.position
		}
		self.hello()
		try:
			self.start()
		except KeyboardInterrupt:
			self.quit(None)
Exemplo n.º 30
0
def get_diff(dirinfo, db_path):
    """Yield entries that are different in the database (or not in it)"""
    with UnQLite(db_path.as_posix()) as db:
        for info in dirinfo:
            if info.name in db:
                dbinfo = json.loads(db[info.name])
                if (info.size != dbinfo['size']
                        and info.mtime != dbinfo['mtime']
                        and info.md5 != dbinfo['md5']):
                    yield info
            else:
                yield info
Exemplo n.º 31
0
def process(bug_reports, repository_path, data_prefix):
    ast_cache = prepare_ast_cache(repository_path)

    ast_cache_collection_db = UnQLite(data_prefix + "_ast_cache_collection_db")

    before = default_timer()
    for k, v in ast_cache.items():
        ast_cache_collection_db[k] = pickle.dumps(v, -1)
    after = default_timer()
    total = after - before
    print("total ast cache saving time ", total)

    bug_report_files = prepare_bug_report_files(repository_path, bug_reports, ast_cache)

    before = default_timer()

    bug_report_files_collection_db = UnQLite(data_prefix + "_bug_report_files_collection_db")
    for k, v in bug_report_files.items():
        bug_report_files_collection_db[k] = pickle.dumps(v, -1)

    after = default_timer()
    total = after - before
    print("total bug report files saving time ", total)
Exemplo n.º 32
0
    def __init__(self, master, local):
        """define  UnQLite database"""
        db = UnQLite()
        """ define collection where we'll insert local files content"""
        self.db_local = db.collection("db_local")
        self.db_local.create()
        """ define collection where we'll insert master files content"""
        self.db_master = db.collection("db_master")
        self.db_master.create()

        with open(local) as bibtex_file:
            bibtex_str = bibtex_file.read()

        """create bibtex database for local file"""
        self.bibdb_local = bibtexparser.loads(bibtex_str)
        self.db_local.store(self.bibdb_local.entries)

        with open(master) as bibtex_file:
            bibtex_str = bibtex_file.read()

        """create bibtex database for master file"""
        bibdb_master = bibtexparser.loads(bibtex_str)
        self.db_master.store(bibdb_master.entries)
Exemplo n.º 33
0
Arquivo: db.py Projeto: napoler/tkitDb
class UnQDb:
    """
    基于UnQLite的nosql数据存贮
    """
    def __init__(self, dbpath='data.db'):
        # UnQLite.__init__(self,dbpath)
        self.db = UnQLite(dbpath)
        self.dbpath = dbpath

    def __del__(self):
        self.db.close()

    def add(self, key, value):
        """
        添加数据
        key ='2eas'
        value={}
        """
        self.db[key] = value
        # print('222')

    def reload(self):
        self.db = UnQLite(self.dbpath)
        pass

    def get(self, key):
        """
        获取数据
        自动转换成字典
        """

        # print('value',value)
        try:
            value = str(self.db[key], "utf-8")
            value = ast.literal_eval(value)
        except:
            return None
            pass
        return value

    def get_all(self):
        with self.db.cursor() as cursor:
            for key, value in cursor:
                yield key, self.get(key)

    def delete(self, key):
        """
        删除数据
        """
        # del self.db[key]
        self.db.delete(key)

    def col(self, key):
        self.col = self.db.collection(key)
        self.col.create()  # Create the collection if it does not exist.
        self.col.exists()
        return self.col
Exemplo n.º 34
0
        def __init__( self, configFileName, dbPath, timeout ):

            self.dnsArray = list()
            self.responseTimes = dict()

            if os.access( configFileName, os.R_OK ) is False:
                print "Can not read "+configFileName+" !"
                exit(-1)

            if os.access( dbPath, os.F_OK ) is False:
                print "Created DB Directory at "+dbPath
                os.mkdir( dbPath )

            db = UnQLite( dbPath+"/history.db" )
            self.dnsDB = db.collection('dnsResponses')

            if ( self.dnsDB.exists() is False ):
                print "Initial DB Created."
                self.dnsDB.create()

            self.recursor = dns.resolver.Resolver()
            self.recursor.lifetime = timeout
            self.recursor.timeout = self.recursor.lifetime
            dns.resolver.override_system_resolver(self.recursor)

            with open(configFileName) as configFile:
                contents = configFile.readlines()
            validNSRegex = re.compile(r'nameserver (1[0-9][0-9]|2[0-4][0-9]|25[0-5]|[0-9][0-9]|[0-9])\.(1[0-9][0-9]|2[0-4][0-9]|25[0-5]|[0-9][0-9]|[0-9])\.(1[0-9][0-9]|2[0-4][0-9]|25[0-5]|[0-9][0-9]|[0-9])\.(1[0-9][0-9]|2[0-4][0-9]|25[0-5]|[0-9][0-9]|[0-9])')

            for line in contents:
                match = validNSRegex.search(line)
                if ( match is not None ):
                    self.dnsArray.append(match.group(1)+"."+match.group(2)+"."+match.group(3)+"."+match.group(4))
                    print "=> Added "+match.group(1)+"."+match.group(2)+"."+match.group(3)+"."+match.group(4)+" into the list."

            self.currentPreferredDNS = self.dnsArray[0]
            return
Exemplo n.º 35
0
def query_by_field_intersect(db: UnQLite, col: str, key: Hashable,
                             value: List[Any]) -> List[dict]:
    script = """
        $zCallback = function($doc) {
            return count(array_intersect($doc[$key], $value)) > 0;
        };

        $data = db_fetch_all($col, $zCallback);
    """

    with db.vm(script) as vm:
        vm["col"] = col
        vm["key"] = key
        vm["value"] = value
        vm.execute()
        return vm["data"]
Exemplo n.º 36
0
def create_user(name, mail, description, tags, active=False, is_admin=False):
    user_col = get_collection('user')
    user_uuid = gen_uuid()
    user = {
        'id': user_uuid,
        'name': name,
        'mail': mail,
        'description': description,
        'tags': tags,
        'active': active,
        'is_admin': is_admin
    }
    user_id = user_col.store(user)
    db = UnQLite(db_file)
    db[user_uuid] = user_id
    return user_uuid
Exemplo n.º 37
0
def query_by_field_eq(db: UnQLite, col: str, key: Hashable,
                      value: Any) -> List[dict]:
    script = """
        $zCallback = function($doc) {
            return $doc[$key] == $value;
        };

        $data = db_fetch_all($col, $zCallback);
    """

    with db.vm(script) as vm:
        vm["col"] = col
        vm["key"] = key
        vm["value"] = value
        vm.execute()
        return vm["data"]
Exemplo n.º 38
0
class XlsxReader(object):

    def __init__(self, filepath):
        self.filepath = filepath
        self.name = os.path.basename(filepath)
        # In memory JSON store
        self.db = UnQLite()
        self.workbook = self.load_from_file(filepath)
        self.data = self.extract_as_python()
        self.to_db()

    def __repr__(self):
        return '<XlsxReader: {}>'.format(self.name)

    @staticmethod
    def load_from_file(filepath):
        # read_only: optimised read, data_only: ignores formulae
        return load_workbook(filename=filepath, read_only=True, data_only=True)

    def extract_as_python(self):
        sheets = self.workbook.worksheets

        sheet_data = {}
        columns_data = {}
        for sheet in sheets:
            sheet_title = sheet.title
            rows, columns = self.extract_sheet(sheet)

            sheet_data[sheet_title] = rows
            columns_data[sheet_title] = columns

        sheet_data.update({'_columns': [columns_data]})
        return sheet_data

    def extract_sheet(self, sheet):
        rows = sheet.rows

        # Assume the first row of the sheet is the columns
        column_cells = next(rows)
        columns = [cell.value for cell in column_cells]

        extracted_rows = []
        for row_idx, row in enumerate(rows, start=1):
            row_dict = {'row': row_idx}

            # Index into the columns list.
            # Assumes the order is always the same.
            for column_idx, cell in enumerate(row):
                cell_value = self.extract_cell(cell)
                row_dict[columns[column_idx]] = cell_value

            extracted_rows.append(row_dict)

        return extracted_rows, columns

    def extract_cell(self, cell):
        """
        Workarounds for OpenPyXL live here.
        If the cell value has a comma, we assume it is a list and split it.
        All dates in Excel are TZ naive, so add one.
        """
        # TODO: test me directly
        val = cell.value
        if isinstance(val, unicode):
            # Takes 'foo', bar'
            # Returns ['foo', 'bar']
            if ',' in val:
                return [i.lstrip() for i in val.split(',')]

        if isinstance(val, datetime):
            # Just assume everything is UTC for the time being.
            return UTC.localize(val)

        return val

    def to_json(self):
        return json.dumps(self.data, default=json_serialiser)

    def to_db(self):
        self.collections = {
            name: self.create_collection(name, documents)
            for name, documents in self.data.items()
        }

    def create_collection(self, name, documents):
        collection = self.db.collection(name)
        collection.create()
        docs = self.pre_store(documents)
        collection.store(docs)
        return collection

    def pre_store(self, documents):
        """
        We can't change UnQLite's json serialiser, so we need to
        check for datetime objs here. We drop the 'row' key as it's of
        no use once stored.

        Also, we try to naively call json.dumps() to raise errors,
        since UnQLite silently stores a None if it can't serialise.
        Doh!
        """
        checked_docs = []
        for doc in documents:

            checked_doc = {}
            for k, v in doc.items():
                if k == 'row':
                    continue
                if isinstance(v, datetime):
                    checked_doc[k] = v.isoformat()
                    continue
                json.dumps(v)
                checked_doc[k] = v

            checked_docs.append(checked_doc)

        return checked_docs
Exemplo n.º 39
0
 def setUp(self):
     super(BaseTestCase, self).setUp()
     self.db = UnQLite(':mem:')
     self._filename = 'test.db'
     self.file_db = UnQLite(self._filename)
__author__ = 'James Veitch'

'''
Example to connect and download all issues on the remote server to a local cache nosql
database using UnQLite
'''

from unqlite import UnQLite
from pprint import pprint

import pythreesixty.core.mobile as mobile

db = UnQLite()  # Creates an in-memory database.
issues = db.collection('issues')
issues.create()  # Create the collection if it does not exist.

projects = mobile.projects()
print("User has access to {count} projects".format(count=len(projects.json())))

for p in projects.json():
    print("\n{id}: {name}".format(
        id=p['project_id'],
        name=p['name'])
    )

    # Get issues
    issues_request = mobile.get_issues(p['project_id'], count_only=True)  # use count_only to just return the number
    if issues_request.status_code == 200:
        print("  - {count} issues found.".format(count=issues_request.json()['count']))

    issues.store(mobile.get_issues(p['project_id']).json())
Exemplo n.º 41
0
# -*- coding: utf-8 -*-
import hashlib
import os
from unqlite import UnQLite
import exifread
import datetime


db = UnQLite('./memories.db')
pictures = db.collection('pictures')
pictures.create()
pictures.store({'name': 'Leslie', 'color': 'also green'})
print pictures.fetch(0)


if __name__ == '__main__':
    for dirpath, dirs, files in os.walk('/home/stephan/Images/2015'):
        for filename in files:
            with open(os.path.join(dirpath, filename), 'rb') as fname:
                exif = exifread.process_file(fname, details=False)
                photo_date = datetime.datetime.strptime(exif['EXIF DateTimeOriginal'].values, '%Y:%m:%d %H:%M:%S')
                resolution = (exif['EXIF ExifImageWidth'].values[0], exif['EXIF ExifImageLength'].values[0])
                inserted = modified = datetime.datetime.now()
                md5 = hashlib.md5(fname.read()).hexdigest()

                print filename
                print photo_date
                print resolution
                print inserted
                print modified
                print md5
Exemplo n.º 42
0
parser = argparse.ArgumentParser(description='Ubicajeros API actualizar db')
parser.add_argument('-r', '--radio', required=False, help='Radio de busqueda en kms.', default='1000')
parser.add_argument('-l', '--latlon', required=False, help='Latlon de busqueda', default='19.432608,-99.133209')

args = parser.parse_args()

CAJEROS_URL = 'http://www.banxico.org.mx/consultas-atm/cajeros.json?l=' + args.latlon + '&b=&r=' + args.radio
CAJERO_URL = 'http://www.banxico.org.mx/consultas-atm/cajeros/info.json'

print 'Buscando cajeros en ' + CAJEROS_URL
cajeros_json = requests.get(CAJEROS_URL).json()['contenido']
total_cajeros = len(cajeros_json)

agregados = 0
for i, cajero_json in enumerate(cajeros_json):
    db = UnQLite('cajeros.db')
    db.open()
    cajero = {}
    cajero['id'] = cajero_json['id']
    cajero['clave_institucion'] = cajero_json['cb']
    cajero['lat'] = cajero_json['l']['lat']
    cajero['lon'] = cajero_json['l']['lng']
    cajero['nombre_institucion'] = NOMBRES[cajero['clave_institucion']]
    try:
        print 'Cajero ' + str(i) + ' de ' + str(total_cajeros) + ', ' + str(cajero['id']) + ' existe? ' + str(db.exists(cajero['id']))
        if not db.exists(cajero['id']):
            url_cajero = CAJERO_URL + '?id=' + str(cajero['id']) + '&banco=' + str(cajero['clave_institucion'])
            cajero_json = requests.get(url_cajero).json()['contenido']
            cajero['cp'] = str(cajero_json['cp'])
            cajero['horario'] = cajero_json['hs']
            cajero['direccion'] = cajero_json['d']
Exemplo n.º 43
0
 def setUp(self):
     self._filename = "test.db"
     self.db = UnQLite(self._filename)