def move_process():
form = MoveProssessForm()
user = users.get_current_user()
process_id = None
if request.method == 'POST':
if form.validate_on_submit():
try:
move_process = MoveProcess(ok_count=0, error_count=0,
total_count=0)
emails = list(set([
email.strip() for email in form.data['emails']
.splitlines()]))
if len(emails) > 0:
move_process.emails = emails
move_process.tag = form.data['tag']
move_process_key = move_process.put()
for email in emails:
deferred.defer(
schedule_user_move,
user_email=email,
tag=move_process.tag,
# TODO: Domain of the namespace
domain_name=secret_keys.OAUTH2_CONSUMER_KEY,
move_process_key=move_process_key
)
process_id = move_process_key.id()
else:
form.errors['Emails'] = ['Emails should not be empty']
except BadValueError, e:
logging.exception('error saving process')
form.errors['Emails'] = ['Emails are malformed']
def finalized(self):
move_process_id = self.kwargs.get('move_process_id')
logging.info('Finishing process [%s]', move_process_id)
process = MoveProcess.get_by_id(move_process_id)
process.status = constants.FINISHED
process.execution_finish = datetime.now()
process.put()
def run(self, message_process_id=None, user_process_id=None,
move_process_id=None):
move_process = MoveProcess.get_by_id(move_process_id)
user_process = MoveUserProcess.get_by_id(user_process_id)
try:
move_message(user_process=user_process,
message_process_id=message_process_id,
label=move_process.tag)
except Exception as e:
if self.current_attempt >= self.max_attempts:
logging.exception(
'Failed definitely moving the message id [%s] for user [%s] messages',
message_process_id, user_process.user_email)
message_process = MoveMessageProcess.get_by_id(
message_process_id)
message_process.status = constants.FAILED
message_process.error_description = e.message
message_process.put()
counter.load_and_increment_counter(
'%s_%s_error_counter' % (
user_process.user_email, user_process.key.id()))
else:
logging.exception(
'Failed retrieving a messagee id [%s] for [%s], '
'try again...', message_process_id, user_process.user_email)
raise e
def run(self, move_process_id):
logging.info("start process for move_process %s", move_process_id)
process = MoveProcess.get_by_id(move_process_id)
emails = process.emails
user_processes = []
for email in emails:
user_process = MoveUserProcess(
user_email=email,
move_process_key=process.key,
status=constants.STARTED
)
user_process_key = user_process.put()
user_process_id = user_process_key.id()
user_processes.append((yield MoveUserProcessPipeline(
user_process_id=user_process_id, tag=process.tag)))
yield List(*user_processes)
def generate_count_report():
# Process counters with the latest syntax
futures = []
processes = MoveProcess.query().fetch()
logging.info('Generating count report in [%s] processes', len(processes))
for process in processes:
process_ok_count = 0
process_error_count = 0
process_total_count = 0
for email in process.emails:
user = ProcessedUser.get_by_id(email)
if not user:
user = ProcessedUser(id=email, ok_count=0, error_count=0,
total_count=list())
total_count = counter.load_and_get_count(
'%s_total_count' % email, namespace=str(process.key.id()))
if total_count:
process_total_count += total_count
else:
total_count = 0
if total_count not in user.total_count:
user.total_count.append(total_count)
ok_count = counter.load_and_get_count(
'%s_ok_count' % email, namespace=str(process.key.id()))
if ok_count:
process_ok_count += ok_count
user.ok_count += ok_count
error_count = counter.load_and_get_count(
'%s_error_count' % email, namespace=str(process.key.id()))
if error_count:
process_error_count += error_count
user.error_count += error_count
futures.append(user.put_async())
logging.info('Updating process counters: total [%s] ok [%s] error ['
'%s]', process_total_count, process_ok_count,
process_error_count)
process.ok_count = process_ok_count
process.error_count = process_error_count
process.total_count = process_total_count
futures.append(process.put_async())
# Process futures
[future.get_result() for future in futures]