def setUp(self):
# config info
self.gnupg_home = "%s/gnupg" % self.tempdir
self.db1_file = "%s/db1.u1db" % self.tempdir
self.db2_file = "%s/db2.u1db" % self.tempdir
self.email = '*****@*****.**'
# open test dbs
self._db1 = u1db.open(self.db1_file,
create=True,
document_factory=LeapDocument)
self._db2 = u1db.open(self.db2_file,
create=True,
document_factory=LeapDocument)
# initialize soledad by hand so we can control keys
self._soledad = Soledad(self.email,
gnupg_home=self.gnupg_home,
initialize=False,
prefix=self.tempdir)
self._soledad._init_dirs()
self._soledad._gpg = GPGWrapper(gnupghome=self.gnupg_home)
self._soledad._gpg.import_keys(PUBLIC_KEY)
self._soledad._gpg.import_keys(PRIVATE_KEY)
self._soledad._load_openpgp_keypair()
if not self._soledad._has_secret():
self._soledad._gen_secret()
self._soledad._load_secret()
self._soledad._init_db()
def setUp(self):
# open test dbs
self.db1_file = os.path.join(
self.tempdir, "db1.u1db")
self.db2_file = os.path.join(
self.tempdir, "db2.u1db")
self._db1 = u1db.open(self.db1_file, create=True,
document_factory=SoledadDocument)
self._db2 = u1db.open(self.db2_file, create=True,
document_factory=SoledadDocument)
# soledad config info
self.email = '*****@*****.**'
secrets_path = os.path.join(
self.tempdir, Soledad.STORAGE_SECRETS_FILE_NAME)
local_db_path = os.path.join(
self.tempdir, Soledad.LOCAL_DATABASE_FILE_NAME)
server_url = ''
cert_file = None
self._soledad = self._soledad_instance(
self.email, '123',
secrets_path=secrets_path,
local_db_path=local_db_path,
server_url=server_url,
cert_file=cert_file)
def setUp(self):
# The following snippet comes from BaseLeapTest.setUpClass, but we
# repeat it here because twisted.trial does not work with
# setUpClass/tearDownClass.
self.old_path = os.environ['PATH']
self.old_home = os.environ['HOME']
self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
self.home = self.tempdir
bin_tdir = os.path.join(self.tempdir, 'bin')
os.environ["PATH"] = bin_tdir
os.environ["HOME"] = self.tempdir
# config info
self.db1_file = os.path.join(self.tempdir, "db1.u1db")
self.db2_file = os.path.join(self.tempdir, "db2.u1db")
self.email = ADDRESS
# open test dbs
self._db1 = u1db.open(self.db1_file,
create=True,
document_factory=SoledadDocument)
self._db2 = u1db.open(self.db2_file,
create=True,
document_factory=SoledadDocument)
# get a random prefix for each test, so we do not mess with
# concurrency during initialization and shutting down of
# each local db.
self.rand_prefix = ''.join(
map(lambda x: random.choice(string.ascii_letters), range(6)))
# initialize soledad by hand so we can control keys
# XXX check if this soledad is actually used
self._soledad = self._soledad_instance(prefix=self.rand_prefix,
user=self.email)
def setUp(self):
# The following snippet comes from BaseLeapTest.setUpClass, but we
# repeat it here because twisted.trial does not work with
# setUpClass/tearDownClass.
self.old_path = os.environ['PATH']
self.old_home = os.environ['HOME']
self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
self.home = self.tempdir
bin_tdir = os.path.join(
self.tempdir,
'bin')
os.environ["PATH"] = bin_tdir
os.environ["HOME"] = self.tempdir
# config info
self.db1_file = os.path.join(self.tempdir, "db1.u1db")
self.db2_file = os.path.join(self.tempdir, "db2.u1db")
self.email = ADDRESS
# open test dbs
self._db1 = u1db.open(self.db1_file, create=True,
document_factory=SoledadDocument)
self._db2 = u1db.open(self.db2_file, create=True,
document_factory=SoledadDocument)
# get a random prefix for each test, so we do not mess with
# concurrency during initialization and shutting down of
# each local db.
self.rand_prefix = ''.join(
map(lambda x: random.choice(string.ascii_letters), range(6)))
# initialize soledad by hand so we can control keys
# XXX check if this soledad is actually used
self._soledad = self._soledad_instance(
prefix=self.rand_prefix, user=self.email)
def get_database():
"""Get the path that the database is stored in."""
# setting document_factory to Task means that any database method that
# would return documents, now returns Tasks instead.
return u1db.open(os.path.join(save_data_path("cosas"), "cosas.u1db"),
create=True,
document_factory=Task)
def create_and_write(dbname, file_path):
"""
Utility to write files to dbname from file_path.
:param dbname: Name of the database.
:type dbname: str
:param file_path: Absolute path of the directory that contains the
files.
:type file_path: str
"""
# Note: When the os.urandom() data is encoded to be inserted as a
# doc_content in u1db, its size may change depending on the encoding
# chosen. So the code is being benchmarked against larger files than
# what it may appear to be.
db = u1db.open(os.path.join(BASE_DB_PATH, dbname), create=True)
files = os.listdir(file_path)
for f in files:
file_ob = open(os.path.join(file_path, f), "rb")
data = ""
while True:
byte = file_ob.read(45)
if not byte:
break
data += binascii.b2a_uu(byte)
file_ob.close()
db.create_doc({
"name": f,
"data": data
})
docs = db.get_all_docs()
db.close()
def _synchronize(self, creds=None):
if self.u1_radio.isChecked():
# TODO: not hardcode
target = 'https://u1db.one.ubuntu.com/~/u1todo'
else:
target = self.url_edit.text()
if target.startswith('http://') or target.startswith('https://'):
st = HTTPSyncTarget.connect(target)
oauth_creds = {
'token_key': creds['token'],
'token_secret': creds['token_secret'],
'consumer_key': creds['consumer_key'],
'consumer_secret': creds['consumer_secret']}
if creds:
st.set_oauth_credentials(**oauth_creds)
else:
db = u1db.open(target, create=True)
st = db.get_sync_target()
syncer = Synchronizer(self.store.db, st)
try:
syncer.sync()
except DatabaseDoesNotExist:
# The server does not yet have the database, so create it.
if target.startswith('http://') or target.startswith('https://'):
db = HTTPDatabase(target)
db.set_oauth_credentials(**oauth_creds)
db.open(create=True)
syncer.sync()
self.refresh_filter()
self.last_synced.setText(
'<span style="color:green">%s</span>' % (datetime.now()))
self.sync_button.setEnabled(True)
def __init__(self, name, creds=None):
if name == "test.u1db":
db = name
else:
db = os.getcwd() + "/extern/u1db/" + name
self.db = u1db.open(db, create=True)
self.url = settings['Oauth']['sync'] + settings['Oauth']['server'] + ":" + str(settings['Oauth']['port']) + "/" + name
self.creds = creds
def soledad_sync(uuid, token, server):
tempdir = tempfile.mkdtemp()
try:
db = u1db.open(os.path.join(tempdir, '%s.db' % uuid), True)
creds = {'token': {'uuid': uuid, 'token': token}}
db.sync(server, creds=creds, autocreate=False)
finally:
shutil.rmtree(tempdir)
def soledad_sync(uuid, token, server):
tempdir = tempfile.mkdtemp()
try:
db = u1db.open(os.path.join(tempdir, '%s.db' % uuid), True)
creds = {'token': {'uuid': uuid, 'token': token}}
db.sync(server, creds=creds, autocreate=False)
finally:
shutil.rmtree(tempdir)
def setUp(self):
# config info
self.db1_file = os.path.join(self.tempdir, "db1.u1db")
self.db2_file = os.path.join(self.tempdir, "db2.u1db")
self.email = ADDRESS
# open test dbs
self._db1 = u1db.open(self.db1_file, create=True,
document_factory=SoledadDocument)
self._db2 = u1db.open(self.db2_file, create=True,
document_factory=SoledadDocument)
# get a random prefix for each test, so we do not mess with
# concurrency during initialization and shutting down of
# each local db.
self.rand_prefix = ''.join(
map(lambda x: random.choice(string.ascii_letters), range(6)))
# initialize soledad by hand so we can control keys
self._soledad = self._soledad_instance(
prefix=self.rand_prefix, user=self.email)
def __init__(self, name, creds=None,name2 = None):
if name == "test.u1db":
db = name
else:
db = os.getcwd() + "/extern/u1db/" + name
self.db = u1db.open(db, create=True)
db2 = None
if name2 == "test1.u1db":
db2 = name2
elif name2 != None:
db2 = os.getcwd() + "/extern/u1db/" + name2
if db2 != None:
self.db2 = u1db.open(db2, create=True)
#self.url = "http://192.168.3.118:8080/" + name
self.url = settings['Oauth']['sync'] + settings['Oauth']['server'] + ":" + str(settings['Oauth']['port']) + "/" + name
self.creds = creds
def setUp(self):
# config info
self.db1_file = os.path.join(self.tempdir, "db1.u1db")
self.db2_file = os.path.join(self.tempdir, "db2.u1db")
self.email = ADDRESS
# open test dbs
self._db1 = u1db.open(self.db1_file,
create=True,
document_factory=SoledadDocument)
self._db2 = u1db.open(self.db2_file,
create=True,
document_factory=SoledadDocument)
# get a random prefix for each test, so we do not mess with
# concurrency during initialization and shutting down of
# each local db.
self.rand_prefix = ''.join(
map(lambda x: random.choice(string.ascii_letters), range(6)))
# initialize soledad by hand so we can control keys
self._soledad = self._soledad_instance(prefix=self.rand_prefix,
user=self.email)
def __init__(self, name, creds=None, name2=None):
if name == "test.u1db":
db = name
else:
db = os.getcwd() + "/extern/u1db/" + name
self.db = u1db.open(db, create=True)
db2 = None
if name2 == "test1.u1db":
db2 = name2
elif name2 != None:
db2 = os.getcwd() + "/extern/u1db/" + name2
if db2 != None:
self.db2 = u1db.open(db2, create=True)
#self.url = "http://192.168.3.118:8080/" + name
self.url = settings['Oauth']['sync'] + settings['Oauth'][
'server'] + ":" + str(settings['Oauth']['port']) + "/" + name
self.creds = creds
def __init__(self, name, creds=None):
if name == "test_v1.u1db":
db = name
else:
db = os.getcwd() + "/extern/u1db/" + name
self.db = u1db.open(db, create=True)
# self.url = "http://192.168.3.118:8080/" + name
self.url = (
settings["Oauth"]["sync"] + settings["Oauth"]["server"] + ":" + str(settings["Oauth"]["port"]) + "/" + name
)
self.creds = creds
def can_sync_soledad_fine():
tempdir = tempfile.mkdtemp()
try:
uuid, server_url, cert_file, token = \
get_soledad_info(Config(), tempdir)
# in the future, we can replace the following by an actual Soledad
# client sync, if needed
db = u1db.open(os.path.join(tempdir, '%s.db' % uuid), True)
creds = {'token': {'uuid': uuid, 'token': token}}
db.sync(server_url, creds=creds, autocreate=False)
finally:
shutil.rmtree(tempdir)
def can_sync_soledad_fine():
tempdir = tempfile.mkdtemp()
try:
uuid, server_url, cert_file, token = \
get_soledad_info(Config(), tempdir)
# in the future, we can replace the following by an actual Soledad
# client sync, if needed
db = u1db.open(os.path.join(tempdir, '%s.db' % uuid), True)
creds = {'token': {'uuid': uuid, 'token': token}}
db.sync(server_url, creds=creds, autocreate=False)
finally:
shutil.rmtree(tempdir)
def setUp(self):
# config info
self.gnupg_home = "%s/gnupg" % self.tempdir
self.db1_file = "%s/db1.u1db" % self.tempdir
self.db2_file = "%s/db2.u1db" % self.tempdir
self.email = '*****@*****.**'
# open test dbs
self._db1 = u1db.open(self.db1_file, create=True,
document_factory=LeapDocument)
self._db2 = u1db.open(self.db2_file, create=True,
document_factory=LeapDocument)
# initialize soledad by hand so we can control keys
self._soledad = Soledad(self.email, gnupg_home=self.gnupg_home,
initialize=False,
prefix=self.tempdir)
self._soledad._init_dirs()
self._soledad._gpg = GPGWrapper(gnupghome=self.gnupg_home)
self._soledad._gpg.import_keys(PUBLIC_KEY)
self._soledad._gpg.import_keys(PRIVATE_KEY)
self._soledad._load_openpgp_keypair()
if not self._soledad._has_secret():
self._soledad._gen_secret()
self._soledad._load_secret()
self._soledad._init_db()
def makeEmptyRepo(repoDir):
db = u1db.open(os.path.join(repoDir, 'papers.u1db'), True)
db.close()
tags = DataModel.TagTree()
tags['Stats']
tags['Stats']['Monte Carlo']
tags['Stats']['Bayesian Inference']
tags['Important']
tags['My Work']
f=open(os.path.join(repoDir, 'tags.json'), 'w')
f.write(tags.toJson())
f.close()
os.mkdir(os.path.join(repoDir, 'repo'))
os.mkdir(os.path.join(repoDir, 'mnt'))
def open_source_db(cls=None, create=True):
"""
Helper method to open the database representing the source replica.
:param cls: A class instance. Default is None. If it is set to None,
then init_server_state returns a server_state.ServerState
instance.
:type cls: type
:param create: A flag to indicate that the source database will be
created or not.
:type create: bool
:return: None or (Database instance and source_replica_uid)
:rtype: None/tuple
"""
source = u1db.open((os.path.join(DATABASE_ROOT,
"source-USER-1.u1db")), create=create)
source_replica_uid = source._get_replica_uid()
if cls:
cls.source = source
cls.source_replica_uid = source_replica_uid
else:
return source, source_replica_uid
def _dictify(obj):
new_dict = {}
for key in obj.keys():
entry = obj[key]
new_dict[key] = normalize(entry)
return new_dict
if __name__ == "__main__":
import json
import u1db
# fetching imdb infos
imdb_db = imdb.IMDb()
imdb_id = "0325980"
movie = imdb_db.get_movie(imdb_id)
# normalize movie
nm = normalize(movie)
# connect to u1db and save doc
db = u1db.open(":memory:", create=True)
db.create_doc(json.dumps(nm), doc_id=imdb_id)
# fetch, load and print doc
restored_doc = db.get_doc(imdb_id)
print json.loads(restored_doc.content)
print "\n\nok"
def init(self, path):
self.db = u1db.open(os.path.join(self.repoDir, 'papers.u1db'),
create=False)
self._init_data()
kwargs['tags'] = tags
if not row['path'] is None:
pdfpath = os.path.join(basedir, row['path'])
pdfpath, _ = importFile(pdfpath, os.path.join(repoDir, 'repo'))
kwargs['path'] = pdfpath
p = DataModel.Paper(**kwargs)
print p
db.create_doc(p.toDict())
repoDir = os.getcwd()
conn = sqlite3.connect(os.path.join(basedir, 'lib.papers'))
db = u1db.open(os.path.join(repoDir, 'papers.u1db'), create=True)
grp_tags = importGroups(conn)
ftags = open('tags.pkl', 'w')
pickle.dump(grp_tags, ftags)
ftags.close()
importPapers(conn, db, grp_tags)
db.create_index('by-author-name',
'''combine(split_words(lower(authors)),
split_words(lower(authors.lastname))
)''')
db.create_index('by-title-words', 'split_words(lower(title))')
from zmq_transport.common.errors import UserIDNotSet
from zmq_transport.config import settings
from zmq_transport.u1db.zmq_target import ZMQSyncTarget
if __name__ == "__main__":
sync_client = ZMQSyncTarget([
settings.ENDPOINT_CLIENT_HANDLER, settings.ENDPOINT_PUBLISHER
])
sync_client.set_user_id("USER-1")
try:
sync_client.start()
except UserIDNotSet as e:
print "Aborting:", e
sys.exit()
source = u1db.open(
os.path.join(settings.DATABASE_ROOT, "source-USER-1.u1db"),
create=False
)
synchronizer = ZMQSynchronizer(source, sync_client)
import time
start = time.time()
source_current_gen = synchronizer.sync()
end = time.time()
print "Sync Completed: ", source_current_gen
print "Total time: ", end - start
"""
Utility script to create database for running tests.
"""
# System imports
import os
import sys
# Dependencies' imports
import u1db
# Local imports
from zmq_transport.config.settings import DATABASE_ROOT
sys.stdout.write('Creating test databases...\n')
source = u1db.open(os.path.join(DATABASE_ROOT, "source-USER-1.u1db"),
create=True)
source.close()
target = u1db.open(os.path.join(DATABASE_ROOT, "user-USER-1.u1db"),
create=True)
target.close()
class ServerThread(threading.Thread):
"""
Thread to run the http server.
"""
def __init__(self, host, port, working_dir, accept_cors_connection=None):
threading.Thread.__init__(self)
self.server = make_server(host, port, working_dir,
accept_cors_connection)
def run(self):
self.server.serve_forever()
def stop(self):
if self.isAlive():
self.server.shutdown()
self.join()
if __name__ == "__main__":
server_thread = ServerThread("127.0.0.1", 9999, DATABASE_ROOT)
server_thread.start()
source_db = u1db.open(os.path.join(BASE_DB_PATH, "source-USER-1.u1db"),
False)
start = time.time()
source_db.sync("http://127.0.0.1:9999/user-USER-1.u1db")
print "Total time: ", time.time() - start
server_thread.stop()
def get_user_db():
hamster_dir = _get_hamster_dir()
db_file = os.path.join(hamster_dir, "hamster.db")
db = u1db.open(db_file, create=True)
return db
def __init__(self, *args, **kwargs):
GObject.__init__(self, *args, **kwargs)
self._connection = u1db.open(":memory:", create=True)
def get_database():
"""Get the path that the database is stored in."""
return u1db.open(
os.path.join(xdg.BaseDirectory.save_data_path("u1todo"),
"u1todo.u1db"), create=True)
return new_list
def _dictify(obj):
new_dict = {}
for key in obj.keys():
entry = obj[key]
new_dict[key] = normalize(entry)
return new_dict
if __name__ == "__main__":
import json
import u1db
# fetching imdb infos
imdb_db = imdb.IMDb()
imdb_id = "0325980"
movie = imdb_db.get_movie(imdb_id)
# normalize movie
nm = normalize(movie)
# connect to u1db and save doc
db = u1db.open(":memory:", create=True)
db.create_doc(json.dumps(nm), doc_id=imdb_id)
# fetch, load and print doc
restored_doc = db.get_doc(imdb_id)
print json.loads(restored_doc.content)
print "\n\nok"
def __init__(self, *args, **kwargs):
GObject.__init__(self, *args, **kwargs)
self._connection = u1db.open(":memory:", create=True)
def get_user_db():
hamster_dir = _get_hamster_dir()
db_file = os.path.join(hamster_dir, "hamster.db")
db = u1db.open(db_file, create=True)
return db
def __init__(self, repoDir, parent=None):
self.repoDir = repoDir
super(DlgImport, self).__init__(parent)
self.matchedPaper = self.MatchedPaper()
self.db = u1db.open(os.path.join(self.repoDir, 'papers.u1db'), create=False)
self.setWindowTitle('Import From PDF Files')
dlgbox = QVBoxLayout()
self.setLayout(dlgbox)
hbox = QHBoxLayout()
dlgbox.addLayout(hbox)
# left pane
vbox = QVBoxLayout()
hbox.addLayout(vbox)
hbox.setStretchFactor(vbox, 1)
hbFile = QHBoxLayout()
vbox.addLayout(hbFile)
self._btOpenFiles = QPushButton('Open Files')
hbFile.addWidget(self._btOpenFiles)
hbFile.addStretch()
self._lstFiles = QListWidget()
self._lstFiles.setMaximumWidth(400)
self._lstFiles.setMinimumHeight(600)
vbox.addWidget(self._lstFiles)
# right pane
vbox = QVBoxLayout()
hbox.addLayout(vbox)
hbox.setStretchFactor(vbox, 5)
# search box
hbSearch = QHBoxLayout()
self._btSearch = QPushButton('Search')
hbSearch.addWidget(self._btSearch)
self._leSearch = QLineEdit()
self._leSearch.setMinimumWidth(600)
hbSearch.addWidget(self._leSearch)
vbox.addLayout(hbSearch)
# search result
self._tbMatches = QTableWidget()
self._tbMatches.setMinimumWidth(600)
vbox.addWidget(self._tbMatches)
self._tePDF = QPlainTextEdit()
vbox.addWidget(self._tePDF)
hbedit = QHBoxLayout()
vbox.addLayout(hbedit)
vbtag = QVBoxLayout()
# Tags
hbedit.addLayout(vbtag)
self._trTags = QTreeWidget()
self._trTags.setMaximumWidth(200)
self._trTags.setSelectionMode(QAbstractItemView.ExtendedSelection)
vbtag.addWidget(self._trTags)
self._btMatch = QPushButton('Match')
self._btSave = QPushButton('Save')
vbtag.addWidget(self._btMatch)
vbtag.addWidget(self._btSave)
# edit json
self._teJson = QPlainTextEdit()
self._teJson.setLineWrapMode(QPlainTextEdit.NoWrap)
hbedit.addWidget(self._teJson)
bbox = QDialogButtonBox(QDialogButtonBox.Close)
dlgbox.addWidget(bbox)
self.connect(self._btOpenFiles, SIGNAL('clicked()'), self.openFiles)
self.connect(self._lstFiles, SIGNAL('itemActivated(QListWidgetItem*)'), self.extractFile)
self.connect(self._btSearch, SIGNAL('clicked()'), self.searchGoogle)
self.connect(self._leSearch, SIGNAL('returnPressed()'), self.searchGoogle)
self.connect(self._btMatch, SIGNAL('clicked()'), self.selectMatch)
self.connect(self._btSave, SIGNAL('clicked()'), self.savePaper)
self.connect(bbox, SIGNAL('rejected()'), self.reject)
self.connect(self, SIGNAL('rejected()'), self.onQuit)
self.initMatchTable()
self.getTags()
self.browser = GScholar.Browser()
def setUp(self):
self.sync_target = ZMQSyncTarget([ENDPOINT_CLIENT_HANDLER,
ENDPOINT_PUBLISHER])
self.source = u1db.open(os.path.join(
DATABASE_ROOT, "source-USER-1.u1db"), create=False)
self.synchronizer = ZMQSynchronizer(self.source, self.sync_target)
