def importer_process_worker(iid):
"""Do the real import work
:param iid: import id we need to pull and work on
"""
logger = importer_process_worker.get_logger()
trans = transaction.begin()
initialize_sql(ini)
import_job = ImportQueueMgr.get(iid)
logger.info("IMPORT: RUNNING for {username}".format(**dict(import_job)))
try:
# process the file using the import script
import_file = open(import_job.file_path)
importer = Importer(
import_file,
import_job.username)
importer.process()
import_job.mark_done()
logger.info(
"IMPORT: COMPLETE for {username}".format(**dict(import_job)))
except Exception, exc:
# we need to log this and probably send an error email to the
# admin
logger = importer_process_worker.get_logger()
logger.error(str(exc))
import_job.mark_error()
def test_import_process(self):
"""Verify importer inserts the correct records"""
good_file = self._get_del_file()
imp = Importer(good_file, username="******")
imp.process()
# now let's do some db sanity checks
_delicious_data_test()
def test_import_process(self):
"""Verify importer inserts the correct records"""
good_file = self._get_del_file()
imp = Importer(good_file, username="******")
imp.process()
# now let's do some db sanity checks
_delicious_data_test()
def test_import_process(self):
"""Verify importer inserts the correct google bookmarks"""
good_file = self._get_google_file()
imp = Importer(good_file, username="******")
imp.process()
# now let's do some db sanity checks
_google_data_test()
def test_import_process(self):
"""Verify importer inserts the correct google bookmarks"""
good_file = self._get_google_file()
imp = Importer(good_file)
imp.process()
# now let's do some db sanity checks
_google_data_test()
def test_bookmarklet_file(self):
"""Verify we can import a file with a bookmarklet in it."""
loc = os.path.dirname(__file__)
bmarklet_file = os.path.join(loc, 'bookmarklet_error.htm')
fh = open(bmarklet_file)
imp = Importer(fh, username=u"admin")
imp.process()
res = Bmark.query.all()
self.assertEqual(len(res), 3)
def test_bookmarklet_file(self):
"""Verify we can import a file with a bookmarklet in it."""
loc = os.path.dirname(__file__)
bmarklet_file = os.path.join(loc, 'bookmarklet_error.htm')
fh = open(bmarklet_file)
imp = Importer(fh, username=u"admin")
imp.process()
res = Bmark.query.all()
self.assertEqual(len(res), 3)
def import_bmarks(request):
"""Allow users to upload a delicious bookmark export"""
data = {}
post = request.POST
LOG.error(request.registry.settings.get('api_key', ''))
LOG.error(post.get('api_key'))
if post:
# we have some posted values
with Authorize(request.registry.settings.get('api_key', ''),
post.get('api_key', None)):
# if auth fails, it'll raise an HTTPForbidden exception
files = post.get('import_file', None)
if files is not None:
# upload is there for use
# process the file using the import script
importer = Importer(files.file)
# we want to store fulltext info so send that along to the
# import processor
conn_str = request.registry.settings.get('sqlalchemy.url',
False)
searcher = get_fulltext_handler(conn_str)
importer.process(fulltext=searcher)
# @todo get a count of the imported bookmarks and setup a flash
# message. Forward to / and display the import message
# request.session.flash("Error something")
return HTTPFound(location=request.route_url('home'))
else:
msg = request.session.pop_flash()
if msg:
data['error'] = msg
else:
data['error'] = None
return data
else:
# just display the form
return {}
def import_bmarks(request):
"""Allow users to upload a delicious bookmark export"""
data = {}
post = request.POST
LOG.error(request.registry.settings.get('api_key', ''))
LOG.error(post.get('api_key'))
if post:
# we have some posted values
with Authorize(request.registry.settings.get('api_key', ''),
post.get('api_key', None)):
# if auth fails, it'll raise an HTTPForbidden exception
files = post.get('import_file', None)
if files is not None:
# upload is there for use
# process the file using the import script
importer = Importer(files.file)
# we want to store fulltext info so send that along to the
# import processor
conn_str = request.registry.settings.get(
'sqlalchemy.url', False)
searcher = get_fulltext_handler(conn_str)
importer.process(fulltext=searcher)
# @todo get a count of the imported bookmarks and setup a flash
# message. Forward to / and display the import message
# request.session.flash("Error something")
return HTTPFound(location=request.route_url('home'))
else:
msg = request.session.pop_flash()
if msg:
data['error'] = msg
else:
data['error'] = None
return data
else:
# just display the form
return {}
def test_dupe_imports(self):
"""If we import twice, we shouldn't end up with duplicate bmarks"""
good_file = self._get_del_file()
imp = Importer(good_file, username=u"admin")
imp.process()
good_file = self._get_del_file()
imp = Importer(good_file, username=u"admin")
imp.process()
# Now let's do some db sanity checks.
self._delicious_xml_data_test()
def test_factory_gives_google(self):
""""Verify that the base importer will give GBookmarkImporter"""
loc = os.path.dirname(__file__)
google_file = os.path.join(loc, 'googlebookmarks.html')
with open(google_file) as google_io:
imp = Importer(google_io, username=u"admin")
self.assertTrue(isinstance(imp, GBookmarkImporter),
"Instance should be a GBookmarkImporter instance")
def test_factory_gives_delicious(self):
""""Verify that the base importer will give DelImporter"""
loc = os.path.dirname(__file__)
del_file = os.path.join(loc, 'delicious.html')
with open(del_file) as del_io:
imp = Importer(del_io, username=u"admin")
self.assertTrue(isinstance(imp, DelImporter),
"Instance should be a delimporter instance")
def import_bmarks(request):
"""Allow users to upload a delicious bookmark export"""
rdict = request.matchdict
username = rdict.get("username")
# if auth fails, it'll raise an HTTPForbidden exception
with ReqAuthorize(request):
data = {}
post = request.POST
if post:
# we have some posted values
files = post.get("import_file", None)
if files is not None:
# upload is there for use
# process the file using the import script
importer = Importer(files.file, username=username)
importer.process()
# @todo get a count of the imported bookmarks and setup a flash
# message. Forward to / and display the import message
# request.session.flash("Error something")
return HTTPFound(location=request.route_url("user_home", username=username))
else:
msg = request.session.pop_flash()
if msg:
data["error"] = msg
else:
data["error"] = None
return data
else:
# just display the form
return {}
def test_dupe_imports(self):
"""If we import twice, we shouldn't end up with duplicate bmarks"""
good_file = self._get_del_file()
imp = Importer(good_file, username=u"admin")
imp.process()
good_file = self._get_del_file()
imp = Importer(good_file, username=u"admin")
imp.process()
# Now let's do some db sanity checks.
self._delicious_xml_data_test()
def importer_process_worker(import_id):
"""Do the real import work
:param import_id: import id we need to pull and work on
"""
trans = transaction.begin()
import_job = ImportQueueMgr.get(import_id)
logger.info("IMPORT: RUNNING for {username}".format(**dict(import_job)))
try:
# process the file using the import script
import_file = open(import_job.file_path)
importer = Importer(
import_file,
import_job.username)
importer.process()
# Processing kills off our transaction so we need to start a new one
# to update that our import is complete.
trans = transaction.begin()
import_job = ImportQueueMgr.get(import_id)
import_job.mark_done()
user = UserMgr.get(username=import_job.username)
from bookie.lib.message import UserImportSuccessMessage
msg = UserImportSuccessMessage(
user.email,
'Bookie: Your requested import has completed.',
INI)
msg.send({
'username': import_job.username,
})
logger.info(
"IMPORT: COMPLETE for {username}".format(**dict(import_job)))
trans.commit()
except Exception, exc:
# We need to log this and probably send an error email to the
# admin
from bookie.lib.message import ImportFailureMessage
from bookie.lib.message import UserImportFailureMessage
trans = transaction.begin()
import_job = ImportQueueMgr.get(import_id)
user = UserMgr.get(username=import_job.username)
msg = ImportFailureMessage(
INI.get('email.from'),
'Import failure!',
INI)
msg.send({
'username': import_job.username,
'file_path': import_job.file_path,
'exc': str(exc)
})
# Also send an email to the user that their import failed.
msg = UserImportFailureMessage(
user.email,
'Bookie: We are sorry, your import failed.',
INI)
msg.send({
'username': import_job.username,
'exc': str(exc)
})
logger.error(exc)
logger.error(str(exc))
import_job.mark_error()
logger.info(
"IMPORT: ERROR for {username}".format(**dict(import_job)))
logger.info(exc)
trans.commit()
def test_doesnt_implement_can_handle(self):
"""Verify we get the exception expected when running can_handle"""
Importer.can_handle("")
def test_doesnt_implement_process(self):
"""Verify we get the exception expected when running process"""
some_io = StringIO.StringIO()
imp = Importer(some_io)
imp.process()
def test_doesnt_implement_can_handle(self):
"""Verify we get the exception expected when running can_handle"""
Importer.can_handle("")
def test_doesnt_implement_process(self):
"""Verify we get the exception expected when running process"""
some_io = StringIO.StringIO()
imp = Importer(some_io)
imp.process()
def test_doesnt_implement_process(self):
"""Verify we get the exception expected when running process"""
some_io = StringIO.StringIO()
imp = Importer(some_io)
self.assertRaises(NotImplementedError, imp.process)