def import_bmarks(self):
"""Allow users to upload a bookmark export file for processing"""
username = self.matchdict.get('username')
# if auth fails, it'll raise an HTTPForbidden exception
with ReqAuthorize(self.request):
data = {}
post = self.POST
# We can't let them submit multiple times, check if this user has
# an import in process.
if ImportQueueMgr.get(username=username, status=NEW):
# They have an import, get the information about it and shoot
# to the template.
return {
'existing': True,
'import_stats': ImportQueueMgr.get_details(
username=username)
}
if post:
# we have some posted values
files = post.get('import_file', None)
if files is not None:
storage_dir_tpl = self.settings.get('import_files',
'/tmp/bookie')
storage_dir = storage_dir_tpl.format(
here=self.settings.get('app_root'))
out_fname = store_import_file(storage_dir, username, files)
# Mark the system that there's a pending import that needs
# to be completed
q = ImportQueue(username, unicode(out_fname))
DBSession.add(q)
DBSession.flush()
# Schedule a task to start this import job.
tasks.importer_process.delay(q.id)
return HTTPFound(
location=self.request.route_url('user_import',
username=username))
else:
msg = self.request.session.pop_flash()
if msg:
data['error'] = msg
else:
data['error'] = None
return data
else:
# we need to see if they've got
# just display the form
return {
'existing': False
}
def importer_process_worker(iid):
"""Do the real import work
:param iid: import id we need to pull and work on
"""
logger = importer_process_worker.get_logger()
trans = transaction.begin()
initialize_sql(ini)
import_job = ImportQueueMgr.get(iid)
logger.info("IMPORT: RUNNING for {username}".format(**dict(import_job)))
try:
# process the file using the import script
import_file = open(import_job.file_path)
importer = Importer(import_file, import_job.username)
importer.process()
import_job.mark_done()
logger.info(
"IMPORT: COMPLETE for {username}".format(**dict(import_job)))
except Exception, exc:
# we need to log this and probably send an error email to the
# admin
logger = importer_process_worker.get_logger()
logger.error(str(exc))
import_job.mark_error()
def importer_depth_rrd():
"""Add these counts to the rrd graph"""
rrd = ImportQueueDepth(
ini.get('rrd_data').format(here=HERE),
ini.get('rrd_graphs').format(here=HERE))
rrd.mark(datetime.now(), ImportQueueMgr.size())
rrd.update()
def importer_process_worker(iid):
"""Do the real import work
:param iid: import id we need to pull and work on
"""
logger = importer_process_worker.get_logger()
trans = transaction.begin()
initialize_sql(ini)
import_job = ImportQueueMgr.get(iid)
logger.info("IMPORT: RUNNING for {username}".format(**dict(import_job)))
try:
# process the file using the import script
import_file = open(import_job.file_path)
importer = Importer(
import_file,
import_job.username)
importer.process()
import_job.mark_done()
logger.info(
"IMPORT: COMPLETE for {username}".format(**dict(import_job)))
except Exception, exc:
# we need to log this and probably send an error email to the
# admin
logger = importer_process_worker.get_logger()
logger.error(str(exc))
import_job.mark_error()
def test_skip_running(self):
"""Verify that if running, it won't get returned again"""
self._login_admin()
res = self._upload()
eq_(res.status, "302 Found",
msg='Import status is 302 redirect by home, ' + res.status)
# now verify that we've got our record
imp = ImportQueueMgr.get_ready()
imp = imp[0]
imp.status=2
DBSession.flush()
imp = ImportQueueMgr.get_ready()
ok_(not imp, 'We should get no results back')
def import_list(request):
"""Provide some import related data."""
import_list = ImportQueueMgr.get_list()
ret = {
'count': len(import_list),
'imports': [dict(h) for h in import_list],
}
return ret
def test_skip_running(self):
"""Verify that if running, it won't get returned again"""
self._login_admin()
res = self._upload()
eq_(res.status,
"302 Found",
msg='Import status is 302 redirect by home, ' + res.status)
# now verify that we've got our record
imp = ImportQueueMgr.get_ready()
imp = imp[0]
imp.status = 2
DBSession.flush()
imp = ImportQueueMgr.get_ready()
ok_(not imp, 'We should get no results back')
def import_list(request):
"""Provide some import related data."""
import_list = ImportQueueMgr.get_list()
ret = {
'count': len(import_list),
'imports': [dict(h) for h in import_list],
}
return _api_response(request, ret)
def importer_depth_rrd():
"""Add these counts to the rrd graph"""
rrd = ImportQueueDepth(
ini.get('rrd_data').format(here=HERE),
ini.get('rrd_graphs').format(here=HERE))
rrd.mark(
datetime.now(),
ImportQueueMgr.size()
)
rrd.update()
def importer_process():
"""Check for new imports that need to be scheduled to run"""
initialize_sql(ini)
imports = ImportQueueMgr.get_ready(limit=1)
for i in imports:
# Log that we've scheduled it
logger = importer_process.get_logger()
logger.info("IMPORT: SCHEDULED for {username}.".format(**dict(i)))
# We need to mark that it's running to prevent it getting picked up
# again.
trans = transaction.begin()
i.mark_running()
trans.commit()
celery.task.subtask(importer_process_worker, args=(i.id,)).delay()
def importer_process():
"""Check for new imports that need to be scheduled to run"""
initialize_sql(ini)
imports = ImportQueueMgr.get_ready(limit=1)
for i in imports:
# Log that we've scheduled it
logger = importer_process.get_logger()
logger.info("IMPORT: SCHEDULED for {username}.".format(**dict(i)))
# We need to mark that it's running to prevent it getting picked up
# again.
trans = transaction.begin()
i.mark_running()
trans.commit()
celery.task.subtask(importer_process_worker, args=(i.id, )).delay()
def import_reset(request):
"""Reset an import to try again"""
rdict = request.matchdict
import_id = rdict.get('id', None)
if not id:
request.response.status_int = 400
ret = {'error': "Bad request, missing parameters"}
return ret
imp = ImportQueueMgr.get(int(import_id))
imp.status = 0
tasks.importer_process.delay(imp.id)
ret = {'import': dict(imp)}
return ret
def import_reset(request):
"""Reset an import to try again"""
rdict = request.matchdict
import_id = rdict.get('id', None)
if not id:
request.response.status_int = 400
ret = {'error': "Bad request, missing parameters"}
return _api_response(request, ret)
imp = ImportQueueMgr.get(int(import_id))
imp.status = 0
tasks.importer_process.delay(imp.id)
ret = {
'import': dict(imp)
}
return _api_response(request, ret)
def importer_process(import_id):
"""Start the process of running the import.
We load it, mark it as running, and begin begin a task to process.
:param import_id: import id we need to pull and work on
"""
trans = transaction.begin()
imp = ImportQueueMgr.get(import_id)
import_id = imp.id
# Log that we've scheduled it
logger.info("IMPORT: SCHEDULED for {0}.".format(imp.username))
# We need to mark that it's running to prevent it getting picked up
# again.
imp.mark_running()
trans.commit()
importer_process_worker.delay(import_id)
def test_import_upload(self):
"""After we upload a file, we should have an importer queue."""
self._login_admin()
# verify we get the form
res = self.app.get('/admin/import')
ok_('<form' in res.body,
'Should have a form in the body for submitting the upload')
res = self._upload()
eq_(res.status, "302 Found",
msg='Import status is 302 redirect by home, ' + res.status)
# now verify that we've got our record
imp = ImportQueueMgr.get_ready()
imp = imp[0]
ok_(imp, 'We should have a record')
ok_(imp.file_path.endswith('admin.delicious.html'))
eq_(imp.status, 0, 'start out as default status of 0')
def test_import_upload(self):
"""After we upload a file, we should have an importer queue."""
self._login_admin()
# verify we get the form
res = self.app.get('/admin/import')
ok_('<form' in res.body,
'Should have a form in the body for submitting the upload')
res = self._upload()
eq_(res.status,
"302 Found",
msg='Import status is 302 redirect by home, ' + res.status)
# now verify that we've got our record
imp = ImportQueueMgr.get_ready()
imp = imp[0]
ok_(imp, 'We should have a record')
ok_(imp.file_path.endswith('admin.delicious.html'))
eq_(imp.status, 0, 'start out as default status of 0')
def importer_process_worker(import_id):
"""Do the real import work
:param import_id: import id we need to pull and work on
"""
trans = transaction.begin()
import_job = ImportQueueMgr.get(import_id)
logger.info("IMPORT: RUNNING for {username}".format(**dict(import_job)))
try:
# process the file using the import script
import_file = open(import_job.file_path)
importer = Importer(
import_file,
import_job.username)
importer.process()
# Processing kills off our transaction so we need to start a new one
# to update that our import is complete.
trans = transaction.begin()
import_job = ImportQueueMgr.get(import_id)
import_job.mark_done()
user = UserMgr.get(username=import_job.username)
from bookie.lib.message import UserImportSuccessMessage
msg = UserImportSuccessMessage(
user.email,
'Bookie: Your requested import has completed.',
INI)
msg.send({
'username': import_job.username,
})
logger.info(
"IMPORT: COMPLETE for {username}".format(**dict(import_job)))
trans.commit()
except Exception, exc:
# We need to log this and probably send an error email to the
# admin
from bookie.lib.message import ImportFailureMessage
from bookie.lib.message import UserImportFailureMessage
trans = transaction.begin()
import_job = ImportQueueMgr.get(import_id)
user = UserMgr.get(username=import_job.username)
msg = ImportFailureMessage(
INI.get('email.from'),
'Import failure!',
INI)
msg.send({
'username': import_job.username,
'file_path': import_job.file_path,
'exc': str(exc)
})
# Also send an email to the user that their import failed.
msg = UserImportFailureMessage(
user.email,
'Bookie: We are sorry, your import failed.',
INI)
msg.send({
'username': import_job.username,
'exc': str(exc)
})
logger.error(exc)
logger.error(str(exc))
import_job.mark_error()
logger.info(
"IMPORT: ERROR for {username}".format(**dict(import_job)))
logger.info(exc)
trans.commit()
def import_bmarks(request):
"""Allow users to upload a delicious bookmark export"""
rdict = request.matchdict
username = rdict.get('username')
# if auth fails, it'll raise an HTTPForbidden exception
with ReqAuthorize(request):
data = {}
post = request.POST
# we can't let them submit multiple times, check if this user has an
# import in process
if ImportQueueMgr.get(username=username, status=NEW):
# they have an import, get the information about it and shoot to
# the template
return {
'existing': True,
'import_stats': ImportQueueMgr.get_details(username=username)
}
if post:
# we have some posted values
files = post.get('import_file', None)
if files is not None:
# save the file off to the temp storage
out_dir = "{storage_dir}/{randdir}".format(
storage_dir=request.registry.settings.get(
'import_files',
'/tmp/bookie').format(
here=request.registry.settings.get('app_root')),
randdir=random.choice(string.letters),
)
# make sure the directory exists
# we create it with parents as well just in case
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
out_fname = "{0}/{1}.{2}".format(
out_dir, username, files.filename)
out = open(out_fname, 'w')
out.write(files.file.read())
out.close()
# mark the system that there's a pending import that needs to
# be completed
q = ImportQueue(username, out_fname)
DBSession.add(q)
DBSession.flush()
# Schedule a task to start this import job.
tasks.importer_process.delay(q.id)
return HTTPFound(location=request.route_url('user_import',
username=username))
else:
msg = request.session.pop_flash()
if msg:
data['error'] = msg
else:
data['error'] = None
return data
else:
# we need to see if they've got
# just display the form
return {
'existing': False
}
def count_importer_depth():
"""Mark how deep the importer queue is at the moment"""
total = ImportQueueMgr.size()
stat = StatBookmark(attrib=IMPORTER_CT, data=total)
DBSession.add(stat)
