def setUp(self): # config info self.gnupg_home = "%s/gnupg" % self.tempdir self.db1_file = "%s/db1.u1db" % self.tempdir self.db2_file = "%s/db2.u1db" % self.tempdir self.email = '*****@*****.**' # open test dbs self._db1 = u1db.open(self.db1_file, create=True, document_factory=LeapDocument) self._db2 = u1db.open(self.db2_file, create=True, document_factory=LeapDocument) # initialize soledad by hand so we can control keys self._soledad = Soledad(self.email, gnupg_home=self.gnupg_home, initialize=False, prefix=self.tempdir) self._soledad._init_dirs() self._soledad._gpg = GPGWrapper(gnupghome=self.gnupg_home) self._soledad._gpg.import_keys(PUBLIC_KEY) self._soledad._gpg.import_keys(PRIVATE_KEY) self._soledad._load_openpgp_keypair() if not self._soledad._has_secret(): self._soledad._gen_secret() self._soledad._load_secret() self._soledad._init_db()
def setUp(self): # open test dbs self.db1_file = os.path.join( self.tempdir, "db1.u1db") self.db2_file = os.path.join( self.tempdir, "db2.u1db") self._db1 = u1db.open(self.db1_file, create=True, document_factory=SoledadDocument) self._db2 = u1db.open(self.db2_file, create=True, document_factory=SoledadDocument) # soledad config info self.email = '*****@*****.**' secrets_path = os.path.join( self.tempdir, Soledad.STORAGE_SECRETS_FILE_NAME) local_db_path = os.path.join( self.tempdir, Soledad.LOCAL_DATABASE_FILE_NAME) server_url = '' cert_file = None self._soledad = self._soledad_instance( self.email, '123', secrets_path=secrets_path, local_db_path=local_db_path, server_url=server_url, cert_file=cert_file)
def setUp(self): # The following snippet comes from BaseLeapTest.setUpClass, but we # repeat it here because twisted.trial does not work with # setUpClass/tearDownClass. self.old_path = os.environ['PATH'] self.old_home = os.environ['HOME'] self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") self.home = self.tempdir bin_tdir = os.path.join(self.tempdir, 'bin') os.environ["PATH"] = bin_tdir os.environ["HOME"] = self.tempdir # config info self.db1_file = os.path.join(self.tempdir, "db1.u1db") self.db2_file = os.path.join(self.tempdir, "db2.u1db") self.email = ADDRESS # open test dbs self._db1 = u1db.open(self.db1_file, create=True, document_factory=SoledadDocument) self._db2 = u1db.open(self.db2_file, create=True, document_factory=SoledadDocument) # get a random prefix for each test, so we do not mess with # concurrency during initialization and shutting down of # each local db. self.rand_prefix = ''.join( map(lambda x: random.choice(string.ascii_letters), range(6))) # initialize soledad by hand so we can control keys # XXX check if this soledad is actually used self._soledad = self._soledad_instance(prefix=self.rand_prefix, user=self.email)
def setUp(self): # The following snippet comes from BaseLeapTest.setUpClass, but we # repeat it here because twisted.trial does not work with # setUpClass/tearDownClass. self.old_path = os.environ['PATH'] self.old_home = os.environ['HOME'] self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") self.home = self.tempdir bin_tdir = os.path.join( self.tempdir, 'bin') os.environ["PATH"] = bin_tdir os.environ["HOME"] = self.tempdir # config info self.db1_file = os.path.join(self.tempdir, "db1.u1db") self.db2_file = os.path.join(self.tempdir, "db2.u1db") self.email = ADDRESS # open test dbs self._db1 = u1db.open(self.db1_file, create=True, document_factory=SoledadDocument) self._db2 = u1db.open(self.db2_file, create=True, document_factory=SoledadDocument) # get a random prefix for each test, so we do not mess with # concurrency during initialization and shutting down of # each local db. self.rand_prefix = ''.join( map(lambda x: random.choice(string.ascii_letters), range(6))) # initialize soledad by hand so we can control keys # XXX check if this soledad is actually used self._soledad = self._soledad_instance( prefix=self.rand_prefix, user=self.email)
def get_database(): """Get the path that the database is stored in.""" # setting document_factory to Task means that any database method that # would return documents, now returns Tasks instead. return u1db.open(os.path.join(save_data_path("cosas"), "cosas.u1db"), create=True, document_factory=Task)
def create_and_write(dbname, file_path): """ Utility to write files to dbname from file_path. :param dbname: Name of the database. :type dbname: str :param file_path: Absolute path of the directory that contains the files. :type file_path: str """ # Note: When the os.urandom() data is encoded to be inserted as a # doc_content in u1db, its size may change depending on the encoding # chosen. So the code is being benchmarked against larger files than # what it may appear to be. db = u1db.open(os.path.join(BASE_DB_PATH, dbname), create=True) files = os.listdir(file_path) for f in files: file_ob = open(os.path.join(file_path, f), "rb") data = "" while True: byte = file_ob.read(45) if not byte: break data += binascii.b2a_uu(byte) file_ob.close() db.create_doc({ "name": f, "data": data }) docs = db.get_all_docs() db.close()
def _synchronize(self, creds=None): if self.u1_radio.isChecked(): # TODO: not hardcode target = 'https://u1db.one.ubuntu.com/~/u1todo' else: target = self.url_edit.text() if target.startswith('http://') or target.startswith('https://'): st = HTTPSyncTarget.connect(target) oauth_creds = { 'token_key': creds['token'], 'token_secret': creds['token_secret'], 'consumer_key': creds['consumer_key'], 'consumer_secret': creds['consumer_secret']} if creds: st.set_oauth_credentials(**oauth_creds) else: db = u1db.open(target, create=True) st = db.get_sync_target() syncer = Synchronizer(self.store.db, st) try: syncer.sync() except DatabaseDoesNotExist: # The server does not yet have the database, so create it. if target.startswith('http://') or target.startswith('https://'): db = HTTPDatabase(target) db.set_oauth_credentials(**oauth_creds) db.open(create=True) syncer.sync() self.refresh_filter() self.last_synced.setText( '<span style="color:green">%s</span>' % (datetime.now())) self.sync_button.setEnabled(True)
def __init__(self, name, creds=None): if name == "test.u1db": db = name else: db = os.getcwd() + "/extern/u1db/" + name self.db = u1db.open(db, create=True) self.url = settings['Oauth']['sync'] + settings['Oauth']['server'] + ":" + str(settings['Oauth']['port']) + "/" + name self.creds = creds
def soledad_sync(uuid, token, server): tempdir = tempfile.mkdtemp() try: db = u1db.open(os.path.join(tempdir, '%s.db' % uuid), True) creds = {'token': {'uuid': uuid, 'token': token}} db.sync(server, creds=creds, autocreate=False) finally: shutil.rmtree(tempdir)
def setUp(self): # config info self.db1_file = os.path.join(self.tempdir, "db1.u1db") self.db2_file = os.path.join(self.tempdir, "db2.u1db") self.email = ADDRESS # open test dbs self._db1 = u1db.open(self.db1_file, create=True, document_factory=SoledadDocument) self._db2 = u1db.open(self.db2_file, create=True, document_factory=SoledadDocument) # get a random prefix for each test, so we do not mess with # concurrency during initialization and shutting down of # each local db. self.rand_prefix = ''.join( map(lambda x: random.choice(string.ascii_letters), range(6))) # initialize soledad by hand so we can control keys self._soledad = self._soledad_instance( prefix=self.rand_prefix, user=self.email)
def __init__(self, name, creds=None,name2 = None): if name == "test.u1db": db = name else: db = os.getcwd() + "/extern/u1db/" + name self.db = u1db.open(db, create=True) db2 = None if name2 == "test1.u1db": db2 = name2 elif name2 != None: db2 = os.getcwd() + "/extern/u1db/" + name2 if db2 != None: self.db2 = u1db.open(db2, create=True) #self.url = "http://192.168.3.118:8080/" + name self.url = settings['Oauth']['sync'] + settings['Oauth']['server'] + ":" + str(settings['Oauth']['port']) + "/" + name self.creds = creds
def setUp(self): # config info self.db1_file = os.path.join(self.tempdir, "db1.u1db") self.db2_file = os.path.join(self.tempdir, "db2.u1db") self.email = ADDRESS # open test dbs self._db1 = u1db.open(self.db1_file, create=True, document_factory=SoledadDocument) self._db2 = u1db.open(self.db2_file, create=True, document_factory=SoledadDocument) # get a random prefix for each test, so we do not mess with # concurrency during initialization and shutting down of # each local db. self.rand_prefix = ''.join( map(lambda x: random.choice(string.ascii_letters), range(6))) # initialize soledad by hand so we can control keys self._soledad = self._soledad_instance(prefix=self.rand_prefix, user=self.email)
def __init__(self, name, creds=None, name2=None): if name == "test.u1db": db = name else: db = os.getcwd() + "/extern/u1db/" + name self.db = u1db.open(db, create=True) db2 = None if name2 == "test1.u1db": db2 = name2 elif name2 != None: db2 = os.getcwd() + "/extern/u1db/" + name2 if db2 != None: self.db2 = u1db.open(db2, create=True) #self.url = "http://192.168.3.118:8080/" + name self.url = settings['Oauth']['sync'] + settings['Oauth'][ 'server'] + ":" + str(settings['Oauth']['port']) + "/" + name self.creds = creds
def __init__(self, name, creds=None): if name == "test_v1.u1db": db = name else: db = os.getcwd() + "/extern/u1db/" + name self.db = u1db.open(db, create=True) # self.url = "http://192.168.3.118:8080/" + name self.url = ( settings["Oauth"]["sync"] + settings["Oauth"]["server"] + ":" + str(settings["Oauth"]["port"]) + "/" + name ) self.creds = creds
def can_sync_soledad_fine(): tempdir = tempfile.mkdtemp() try: uuid, server_url, cert_file, token = \ get_soledad_info(Config(), tempdir) # in the future, we can replace the following by an actual Soledad # client sync, if needed db = u1db.open(os.path.join(tempdir, '%s.db' % uuid), True) creds = {'token': {'uuid': uuid, 'token': token}} db.sync(server_url, creds=creds, autocreate=False) finally: shutil.rmtree(tempdir)
def makeEmptyRepo(repoDir): db = u1db.open(os.path.join(repoDir, 'papers.u1db'), True) db.close() tags = DataModel.TagTree() tags['Stats'] tags['Stats']['Monte Carlo'] tags['Stats']['Bayesian Inference'] tags['Important'] tags['My Work'] f=open(os.path.join(repoDir, 'tags.json'), 'w') f.write(tags.toJson()) f.close() os.mkdir(os.path.join(repoDir, 'repo')) os.mkdir(os.path.join(repoDir, 'mnt'))
def open_source_db(cls=None, create=True): """ Helper method to open the database representing the source replica. :param cls: A class instance. Default is None. If it is set to None, then init_server_state returns a server_state.ServerState instance. :type cls: type :param create: A flag to indicate that the source database will be created or not. :type create: bool :return: None or (Database instance and source_replica_uid) :rtype: None/tuple """ source = u1db.open((os.path.join(DATABASE_ROOT, "source-USER-1.u1db")), create=create) source_replica_uid = source._get_replica_uid() if cls: cls.source = source cls.source_replica_uid = source_replica_uid else: return source, source_replica_uid
def _dictify(obj): new_dict = {} for key in obj.keys(): entry = obj[key] new_dict[key] = normalize(entry) return new_dict if __name__ == "__main__": import json import u1db # fetching imdb infos imdb_db = imdb.IMDb() imdb_id = "0325980" movie = imdb_db.get_movie(imdb_id) # normalize movie nm = normalize(movie) # connect to u1db and save doc db = u1db.open(":memory:", create=True) db.create_doc(json.dumps(nm), doc_id=imdb_id) # fetch, load and print doc restored_doc = db.get_doc(imdb_id) print json.loads(restored_doc.content) print "\n\nok"
def init(self, path): self.db = u1db.open(os.path.join(self.repoDir, 'papers.u1db'), create=False) self._init_data()
kwargs['tags'] = tags if not row['path'] is None: pdfpath = os.path.join(basedir, row['path']) pdfpath, _ = importFile(pdfpath, os.path.join(repoDir, 'repo')) kwargs['path'] = pdfpath p = DataModel.Paper(**kwargs) print p db.create_doc(p.toDict()) repoDir = os.getcwd() conn = sqlite3.connect(os.path.join(basedir, 'lib.papers')) db = u1db.open(os.path.join(repoDir, 'papers.u1db'), create=True) grp_tags = importGroups(conn) ftags = open('tags.pkl', 'w') pickle.dump(grp_tags, ftags) ftags.close() importPapers(conn, db, grp_tags) db.create_index('by-author-name', '''combine(split_words(lower(authors)), split_words(lower(authors.lastname)) )''') db.create_index('by-title-words', 'split_words(lower(title))')
from zmq_transport.common.errors import UserIDNotSet from zmq_transport.config import settings from zmq_transport.u1db.zmq_target import ZMQSyncTarget if __name__ == "__main__": sync_client = ZMQSyncTarget([ settings.ENDPOINT_CLIENT_HANDLER, settings.ENDPOINT_PUBLISHER ]) sync_client.set_user_id("USER-1") try: sync_client.start() except UserIDNotSet as e: print "Aborting:", e sys.exit() source = u1db.open( os.path.join(settings.DATABASE_ROOT, "source-USER-1.u1db"), create=False ) synchronizer = ZMQSynchronizer(source, sync_client) import time start = time.time() source_current_gen = synchronizer.sync() end = time.time() print "Sync Completed: ", source_current_gen print "Total time: ", end - start
""" Utility script to create database for running tests. """ # System imports import os import sys # Dependencies' imports import u1db # Local imports from zmq_transport.config.settings import DATABASE_ROOT sys.stdout.write('Creating test databases...\n') source = u1db.open(os.path.join(DATABASE_ROOT, "source-USER-1.u1db"), create=True) source.close() target = u1db.open(os.path.join(DATABASE_ROOT, "user-USER-1.u1db"), create=True) target.close()
class ServerThread(threading.Thread): """ Thread to run the http server. """ def __init__(self, host, port, working_dir, accept_cors_connection=None): threading.Thread.__init__(self) self.server = make_server(host, port, working_dir, accept_cors_connection) def run(self): self.server.serve_forever() def stop(self): if self.isAlive(): self.server.shutdown() self.join() if __name__ == "__main__": server_thread = ServerThread("127.0.0.1", 9999, DATABASE_ROOT) server_thread.start() source_db = u1db.open(os.path.join(BASE_DB_PATH, "source-USER-1.u1db"), False) start = time.time() source_db.sync("http://127.0.0.1:9999/user-USER-1.u1db") print "Total time: ", time.time() - start server_thread.stop()
def get_user_db(): hamster_dir = _get_hamster_dir() db_file = os.path.join(hamster_dir, "hamster.db") db = u1db.open(db_file, create=True) return db
def __init__(self, *args, **kwargs): GObject.__init__(self, *args, **kwargs) self._connection = u1db.open(":memory:", create=True)
def get_database(): """Get the path that the database is stored in.""" return u1db.open( os.path.join(xdg.BaseDirectory.save_data_path("u1todo"), "u1todo.u1db"), create=True)
return new_list def _dictify(obj): new_dict = {} for key in obj.keys(): entry = obj[key] new_dict[key] = normalize(entry) return new_dict if __name__ == "__main__": import json import u1db # fetching imdb infos imdb_db = imdb.IMDb() imdb_id = "0325980" movie = imdb_db.get_movie(imdb_id) # normalize movie nm = normalize(movie) # connect to u1db and save doc db = u1db.open(":memory:", create=True) db.create_doc(json.dumps(nm), doc_id=imdb_id) # fetch, load and print doc restored_doc = db.get_doc(imdb_id) print json.loads(restored_doc.content) print "\n\nok"
def __init__(self, repoDir, parent=None): self.repoDir = repoDir super(DlgImport, self).__init__(parent) self.matchedPaper = self.MatchedPaper() self.db = u1db.open(os.path.join(self.repoDir, 'papers.u1db'), create=False) self.setWindowTitle('Import From PDF Files') dlgbox = QVBoxLayout() self.setLayout(dlgbox) hbox = QHBoxLayout() dlgbox.addLayout(hbox) # left pane vbox = QVBoxLayout() hbox.addLayout(vbox) hbox.setStretchFactor(vbox, 1) hbFile = QHBoxLayout() vbox.addLayout(hbFile) self._btOpenFiles = QPushButton('Open Files') hbFile.addWidget(self._btOpenFiles) hbFile.addStretch() self._lstFiles = QListWidget() self._lstFiles.setMaximumWidth(400) self._lstFiles.setMinimumHeight(600) vbox.addWidget(self._lstFiles) # right pane vbox = QVBoxLayout() hbox.addLayout(vbox) hbox.setStretchFactor(vbox, 5) # search box hbSearch = QHBoxLayout() self._btSearch = QPushButton('Search') hbSearch.addWidget(self._btSearch) self._leSearch = QLineEdit() self._leSearch.setMinimumWidth(600) hbSearch.addWidget(self._leSearch) vbox.addLayout(hbSearch) # search result self._tbMatches = QTableWidget() self._tbMatches.setMinimumWidth(600) vbox.addWidget(self._tbMatches) self._tePDF = QPlainTextEdit() vbox.addWidget(self._tePDF) hbedit = QHBoxLayout() vbox.addLayout(hbedit) vbtag = QVBoxLayout() # Tags hbedit.addLayout(vbtag) self._trTags = QTreeWidget() self._trTags.setMaximumWidth(200) self._trTags.setSelectionMode(QAbstractItemView.ExtendedSelection) vbtag.addWidget(self._trTags) self._btMatch = QPushButton('Match') self._btSave = QPushButton('Save') vbtag.addWidget(self._btMatch) vbtag.addWidget(self._btSave) # edit json self._teJson = QPlainTextEdit() self._teJson.setLineWrapMode(QPlainTextEdit.NoWrap) hbedit.addWidget(self._teJson) bbox = QDialogButtonBox(QDialogButtonBox.Close) dlgbox.addWidget(bbox) self.connect(self._btOpenFiles, SIGNAL('clicked()'), self.openFiles) self.connect(self._lstFiles, SIGNAL('itemActivated(QListWidgetItem*)'), self.extractFile) self.connect(self._btSearch, SIGNAL('clicked()'), self.searchGoogle) self.connect(self._leSearch, SIGNAL('returnPressed()'), self.searchGoogle) self.connect(self._btMatch, SIGNAL('clicked()'), self.selectMatch) self.connect(self._btSave, SIGNAL('clicked()'), self.savePaper) self.connect(bbox, SIGNAL('rejected()'), self.reject) self.connect(self, SIGNAL('rejected()'), self.onQuit) self.initMatchTable() self.getTags() self.browser = GScholar.Browser()
def setUp(self): self.sync_target = ZMQSyncTarget([ENDPOINT_CLIENT_HANDLER, ENDPOINT_PUBLISHER]) self.source = u1db.open(os.path.join( DATABASE_ROOT, "source-USER-1.u1db"), create=False) self.synchronizer = ZMQSynchronizer(self.source, self.sync_target)