def install_xpis(self, path, profile):
if not os.path.exists(path): return
utils.print(f'Installing xpis in {path}')
for xpi in glob.glob(os.path.join(path, '*.xpi')):
utils.print(f'installing {xpi}')
profile.add_extension(xpi)
def step_impl(context, seconds):
printed = False
timeout = True
for n in range(seconds):
if not context.zotero.execute('return Zotero.BetterBibTeX.TestSupport.autoExportRunning()'):
timeout = False
break
time.sleep(1)
utils.print('.', end='')
printed = True
if printed: utils.print('')
assert (not timeout), 'Auto-export timed out'
def start(self):
self.needs_restart = False
profile = self.create_profile()
shutil.rmtree(os.path.join(profile.path, self.client, 'better-bibtex'), ignore_errors=True)
if self.client == 'zotero':
datadir_profile = '-datadir profile'
else:
utils.print('\n\n** WORKAROUNDS FOR JURIS-M IN PLACE -- SEE https://github.com/Juris-M/zotero/issues/34 **\n\n')
datadir_profile = ''
cmd = f'{shlex.quote(profile.binary)} -P {shlex.quote(profile.name)} -jsconsole -ZoteroDebugText {datadir_profile} {self.redir} {shlex.quote(profile.path + ".log")} 2>&1'
utils.print(f'Starting {self.client}: {cmd}')
self.proc = subprocess.Popen(cmd, shell=True)
utils.print(f'{self.client} started: {self.proc.pid}')
ready = False
self.config.stash()
self.config.timeout = 2
with benchmark(f'starting {self.client}') as bm:
posted = False
for _ in redo.retrier(attempts=120,sleeptime=1):
utils.print('connecting... (%.2fs)' % (bm.elapsed,))
try:
ready = self.execute("""
if (!Zotero.BetterBibTeX) {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not loaded')
return false;
}
if (!Zotero.BetterBibTeX.ready) {
if (typeof Zotero.BetterBibTeX.ready === 'boolean') {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX initialization error')
} else {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not initialized')
}
return false;
}
Zotero.debug('{better-bibtex:debug bridge}: startup: waiting for BetterBibTeX ready...')
await Zotero.BetterBibTeX.ready;
if (testing && !Zotero.Prefs.get('translators.better-bibtex.testing')) throw new Error('translators.better-bibtex.testing not set!')
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX ready!');
return true;
""", testing = self.testing)
if ready: break
except (urllib.error.HTTPError, urllib.error.URLError,socket.timeout):
pass
if bm.elapsed > 2000 and not posted: posted = post_log()
assert ready, f'{self.client} did not start'
self.config.pop()
if self.import_at_start:
self.execute(f'return await Zotero.BetterBibTeX.TestSupport.importFile({json.dumps(self.import_at_start)})')
self.import_at_start = None
def shutdown(self):
if self.proc is None: return
# graceful shutdown
try:
self.execute("""
const appStartup = Components.classes['@mozilla.org/toolkit/app-startup;1'].getService(Components.interfaces.nsIAppStartup);
appStartup.quit(Components.interfaces.nsIAppStartup.eAttemptQuit);
""")
except:
pass
def on_terminate(proc):
utils.print("process {} terminated with exit code {}".format(
proc, proc.returncode))
zotero = psutil.Process(self.proc.pid)
alive = zotero.children(recursive=True)
alive.append(zotero)
for p in alive:
try:
p.terminate()
except psutil.NoSuchProcess:
pass
gone, alive = psutil.wait_procs(alive,
timeout=5,
callback=on_terminate)
if alive:
for p in alive:
utils.print("process {} survived SIGTERM; trying SIGKILL" % p)
try:
p.kill()
except psutil.NoSuchProcess:
pass
gone, alive = psutil.wait_procs(alive,
timeout=5,
callback=on_terminate)
if alive:
for p in alive:
utils.print("process {} survived SIGKILL; giving up" % p)
self.proc = None
assert not running('Zotero')
def start(self):
self.needs_restart = False
profile = self.create_profile()
cmd = f'{shlex.quote(profile.binary)} -P {shlex.quote(profile.name)} -jsconsole -ZoteroDebugText -datadir profile {self.redir} {shlex.quote(profile.path + ".log")} 2>&1'
utils.print(f'Starting {self.client}: {cmd}')
self.proc = subprocess.Popen(cmd, shell=True)
utils.print(f'{self.client} started: {self.proc.pid}')
ready = False
self.config.stash()
self.config.timeout = 2
with benchmark(f'starting {self.client}') as bm:
posted = None
for _ in redo.retrier(attempts=120, sleeptime=1):
utils.print('connecting... (%.2fs)' % (bm.elapsed, ))
try:
ready = self.execute("""
if (!Zotero.BetterBibTeX) {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not loaded')
return false;
}
if (!Zotero.BetterBibTeX.ready) {
if (typeof Zotero.BetterBibTeX.ready === 'boolean') {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX initialization error')
} else {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not initialized')
}
return false;
}
Zotero.debug('{better-bibtex:debug bridge}: startup: waiting for BetterBibTeX ready...')
await Zotero.BetterBibTeX.ready;
if (testing && !Zotero.Prefs.get('translators.better-bibtex.testing')) throw new Error('translators.better-bibtex.testing not set!')
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX ready!');
return true;
""",
testing=self.testing)
if ready: break
except (urllib.error.HTTPError, urllib.error.URLError,
socket.timeout):
pass
if bm.elapsed > 2000 and not posted: posted = PostLog()
if posted:
utils.print(
'connected, but log posted, waiting for upload to finish...'
)
posted.join()
assert ready, f'{self.client} did not start'
self.config.pop()
def load(lib):
lib = HashableDict(lib)
lib.pop('config', None)
lib.pop('version', None)
items = {
item['itemID']: HashableDict(clean_item(item))
for item in lib['items']
}
lib['items'] = sorted(
items.values(),
key=lambda item: item.get('title', '') + '::' + item.__hash__())
for item in lib['items']:
if 'relations' in item: utils.print(str(item['relations']))
if 'collections' not in lib: lib['collections'] = {}
collections = {k: HashableDict(v) for k, v in lib['collections'].items()}
lib['collections'] = [
coll for coll in collections.values()
if not 'parent' in coll or coll['parent'] == False
]
for coll in list(collections.values()):
coll.pop('parent', None)
coll.pop('id', None)
coll.pop('key', None)
if 'collections' not in coll: coll['collections'] = []
for coll in list(collections.values()):
coll['items'] = sorted(
[items[itemID].__hash__() for itemID in coll['items']])
coll['collections'] = [collections[key] for key in coll['collections']]
lib['collections'] = sorted(
[sort_collection(c) for c in lib['collections']],
key=lambda c: c.__hash__())
return lib
def display(self, start, every, stop):
if stop.is_set(): return
utils.print('.', end='')
threading.Timer(every, self.display, [start, every, stop]).start()
def create_profile(self):
profile = Munch(
name='BBTZ5TEST'
)
profile.path = os.path.expanduser(f'~/.{profile.name}')
profile.profiles = {
# 'Linux': os.path.expanduser(f'~/.{self.client}/{self.client}'),
'Linux': os.path.expanduser(f'~/.{self.client}/zotero'),
'Darwin': os.path.expanduser('~/Library/Application Support/' + {'zotero': 'Zotero', 'jurism': 'Juris-M'}[self.client]),
}[platform.system()]
os.makedirs(profile.profiles, exist_ok = True)
beta = ''
if self.beta: beta = '-beta'
profile.binary = {
'Linux': f'/usr/lib/{self.client}{beta}/{self.client}',
'Darwin': f'/Applications/{self.client.title()}{beta}.app/Contents/MacOS/{self.client}',
}[platform.system()]
# create profile
profile.ini = os.path.join(profile.profiles, 'profiles.ini')
ini = configparser.RawConfigParser()
ini.optionxform = str
if os.path.exists(profile.ini): ini.read(profile.ini)
if not ini.has_section('General'): ini.add_section('General')
profile.id = None
for p in ini.sections():
for k, v in ini.items(p):
if k == 'Name' and v == profile.name: profile.id = p
if not profile.id:
free = 0
while True:
profile.id = f'Profile{free}'
if not ini.has_section(profile.id): break
free += 1
ini.add_section(profile.id)
ini.set(profile.id, 'Name', profile.name)
ini.set(profile.id, 'IsRelative', 0)
ini.set(profile.id, 'Path', profile.path)
ini.set(profile.id, 'Default', None)
with open(profile.ini, 'w') as f:
ini.write(f, space_around_delimiters=False)
# layout profile
if self.config.profile:
profile.firefox = webdriver.FirefoxProfile(os.path.join(ROOT, 'test/db', self.config.profile))
profile.firefox.set_preference('extensions.zotero.dataDir', os.path.join(profile.path, self.client))
profile.firefox.set_preference('extensions.zotero.useDataDir', True)
profile.firefox.set_preference('extensions.zotero.translators.better-bibtex.removeStock', False)
else:
profile.firefox = webdriver.FirefoxProfile(os.path.join(FIXTURES, 'profile', self.client))
self.install_xpis(os.path.join(ROOT, 'xpi'), profile.firefox)
self.install_xpis(os.path.join(ROOT, 'other-xpis'), profile.firefox)
if self.config.db: self.install_xpis(os.path.join(ROOT, 'test/db', self.config.db, 'xpis'), profile.firefox)
if self.config.profile: self.install_xpis(os.path.join(ROOT, 'test/db', self.config.profile, 'xpis'), profile.firefox)
profile.firefox.set_preference('extensions.zotero.translators.better-bibtex.testing', self.testing)
profile.firefox.set_preference('extensions.zotero.translators.better-bibtex.workers', self.workers)
profile.firefox.set_preference('extensions.zotero.debug-bridge.password', self.password)
profile.firefox.set_preference('dom.max_chrome_script_run_time', self.config.timeout)
utils.print(f'dom.max_chrome_script_run_time={self.config.timeout}')
with open(os.path.join(os.path.dirname(__file__), 'preferences.toml')) as f:
preferences = toml.load(f)
for p, v in nested_dict_iter(preferences['general']):
profile.firefox.set_preference(p, v)
if self.config.locale == 'fr':
for p, v in nested_dict_iter(preferences['fr']):
profile.firefox.firefox.set_preference(p, v)
if not self.config.first_run:
profile.firefox.set_preference('extensions.zotero.translators.better-bibtex.citekeyFormat', '[auth][shorttitle][year]')
if self.client == 'jurism':
utils.print('\n\n** WORKAROUNDS FOR JURIS-M IN PLACE -- SEE https://github.com/Juris-M/zotero/issues/34 **\n\n')
profile.firefox.set_preference('extensions.zotero.dataDir', os.path.join(profile.path, 'jurism'))
profile.firefox.set_preference('extensions.zotero.useDataDir', True)
profile.firefox.set_preference('extensions.zotero.translators.better-bibtex.removeStock', False)
profile.firefox.update_preferences()
shutil.rmtree(profile.path, ignore_errors=True)
shutil.move(profile.firefox.path, profile.path)
profile.firefox = None
if self.config.db:
self.needs_restart = True
utils.print(f'restarting using {self.config.db}')
dbs = os.path.join(ROOT, 'test', 'db', self.config.db)
if not os.path.exists(dbs): os.makedirs(dbs)
db_zotero = os.path.join(dbs, f'{self.client}.sqlite')
db_zotero_alt = os.path.join(dbs, self.client, f'{self.client}.sqlite')
if not os.path.exists(db_zotero) and not os.path.exists(db_zotero_alt):
urllib.request.urlretrieve(f'https://github.com/retorquere/zotero-better-bibtex/releases/download/test-database/{self.config.db}.zotero.sqlite', db_zotero)
if not os.path.exists(db_zotero): db_zotero = db_zotero_alt
shutil.copy(db_zotero, os.path.join(profile.path, self.client, os.path.basename(db_zotero)))
db_bbt = os.path.join(dbs, 'better-bibtex.sqlite')
db_bbt_alt = os.path.join(dbs, self.client, 'better-bibtex.sqlite')
if not os.path.exists(db_bbt) and not os.path.exists(db_bbt_alt):
urllib.request.urlretrieve(f'https://github.com/retorquere/zotero-better-bibtex/releases/download/test-database/{self.config.db}.better-bibtex.sqlite', db_bbt)
if not os.path.exists(db_bbt): db_bbt = db_bbt_alt
shutil.copy(db_bbt, os.path.join(profile.path, self.client, os.path.basename(db_bbt)))
# remove any auto-exports that may exist
db = sqlite3.connect(os.path.join(profile.path, self.client, os.path.basename(db_bbt)))
ae = None
for (ae,) in db.execute('SELECT data FROM "better-bibtex" WHERE name = ?', [ 'better-bibtex.autoexport' ]):
ae = json.loads(ae)
ae['data'] = []
if ae:
db.execute('UPDATE "better-bibtex" SET data = ? WHERE name = ?', [ json.dumps(ae), 'better-bibtex.autoexport' ])
db.commit()
db.close()
return profile
def on_terminate(proc):
utils.print("process {} terminated with exit code {}".format(proc, proc.returncode))