def email_failure(self, subject, user_id):
try:
record: User = db.session.query(User).filter_by(id=user_id).first()
except SQLAlchemyError as e:
current_app.logger.exception("SQLAlchemyError exception",
exc_info=e)
raise self.retry()
if record is None:
self.update_state('FAILURE',
meta='Could not load database records')
raise Ignore()
record.post_message(
'An error occurred and your email with subject "{subj}" was not sent '
'to all recipients. Please check the email log to determine which instances were '
'sent correctly. You may wish to consult with a system '
'administrator.'.format(subj=subject),
'error',
autocommit=True)
return True
def replace(self, sig):
"""Replace this task, with a new task inheriting the task id.
.. versionadded:: 4.0
Arguments:
sig (~@Signature): signature to replace with.
Raises:
~@Ignore: This is always raised, so the best practice
is to always use ``raise self.replace(...)`` to convey
to the reader that the task won't continue after being replaced.
"""
chord = self.request.chord
if 'chord' in sig.options:
if chord:
chord = sig.options['chord'] | chord
else:
chord = sig.options['chord']
if isinstance(sig, group):
sig |= self.app.tasks['celery.accumulate'].s(index=0).set(
chord=chord,
link=self.request.callbacks,
link_error=self.request.errbacks,
)
chord = None
if self.request.chain:
for t in self.request.chain:
sig |= signature(t, app=self.app)
sig.freeze(self.request.id,
group_id=self.request.group,
chord=chord,
root_id=self.request.root_id)
sig.delay()
raise Ignore('Replaced by new task')
def replace(self, sig):
"""Replace the current task, with a new task inheriting the
same task id.
:param sig: :class:`@signature`
.. versionadded:: 4.0
Note: This will raise :exc:`~@Ignore`, so the best practice
is to always use ``raise self.replace(...)`` to convey
to the reader that the task will not continue after being replaced.
"""
chord = self.request.chord
if 'chord' in sig.options:
if chord:
chord = sig.options['chord'] | chord
else:
chord = sig.options['chord']
if isinstance(sig, group):
sig |= self.app.tasks['celery.accumulate'].s(index=0).set(
chord=chord,
link=self.request.callbacks,
link_error=self.request.errbacks,
)
chord = None
if self.request.chain:
for t in self.request.chain:
sig |= signature(t)
sig.freeze(self.request.id,
group_id=self.request.group,
chord=chord,
root_id=self.request.root_id)
sig.delay()
raise Ignore('Replaced by new task')
def precomputeLR(self, data: Any, combinations, record_id, project):
data = pd.read_json(data)
# TODO: For higher dimensions do we iterate over whi. h will be the dependant variable?
combinations = list(filter(lambda x: len(x) == 2, combinations))
to_process = 2 * len(combinations)
processed = 0
self.update_state(state=STARTED,
meta={
"processed": processed,
"to_process": to_process
})
results = []
for combo in combinations:
subset = data[combo]
dimensions = ",".join(combo)
for result in computeLR(subset, dimensions, record_id):
results.append(result)
processed += 1
self.update_state(state=STARTED,
meta={
"processed": processed,
"to_process": to_process
})
with getSessionScopeFromId(project) as session:
session.add_all(results)
self.update_state(state=SUCCESS,
meta={
"processed": processed,
"to_process": to_process
})
raise Ignore()
def run(self):
idle_drives = TapeDrive.objects.filter(
status=20,
storage_medium__isnull=False,
last_change__lte=timezone.now() - F('idle_time'),
locked=False,
)
if not idle_drives.exists():
raise Ignore()
for drive in idle_drives.iterator():
robot_queue_entry_exists = RobotQueue.objects.filter(
storage_medium=drive.storage_medium,
req_type=20,
status__in=[0, 2]).exists()
if not robot_queue_entry_exists:
RobotQueue.objects.create(
user=User.objects.get(username='******'),
storage_medium=drive.storage_medium,
req_type=20,
status=0,
)
def long_running_task(self, name):
redis_key = "celery-locks-{0}".format(name)
lock = redis_store.lock(redis_key)
has_lock = False
try:
has_lock = lock.acquire(blocking=False)
if has_lock:
label = "Processing {0}".format(name)
self.update_state(state=u"IN PROGRESS", meta=label)
x = 10
while x > 0:
x = x - 1
time.sleep(1)
else:
label = "Task {0} is processed by another worker.".format(name)
self.update_state(state=states.FAILURE, meta=label)
raise Ignore()
finally:
if has_lock:
lock.release()
return "Finished with {0}. Phew!".format(name)
def book_scraper_chaps_update(self, s_from=0, s_to=0):
"""
Create update task for each novel with revisit=True
"""
books = Book.objects.filter(visited=True).exclude(visit_id__iexact='')
interval = 15
for book in books:
if book.chapters_count and book.revisit_id and not book.revisited:
try:
interval += 3
book.revisited = True
book.save()
schedule, created = IntervalSchedule.objects.get_or_create(
every=interval,
period=IntervalSchedule.SECONDS,
)
PeriodicTask.objects.create(
one_off=True,
interval=schedule,
name=f'Update book chapters: {book.title}',
task='novel2read.apps.books.tasks.book_revisit_novel',
args=json.dumps([book.pk]),
kwargs=json.dumps({
's_from': s_from,
's_to': s_to,
}),
)
except Exception as exc:
exc_result = '\n'.join([f'Book: {book.title}', f'{exc}'])
save_celery_result(
task_id=self.request.id,
task_name=self.name,
status=states.FAILURE,
result=exc_result,
traceback=traceback.format_exc(),
)
raise Ignore()
def log_email(self, task_id, msg: Message):
progress_update(task_id, TaskRecord.RUNNING, 80, "Logging email in database...", autocommit=True)
# don't log if we are not on a live email platform
if not current_app.config.get('EMAIL_IS_LIVE', False):
raise Ignore()
try:
log = None
# store message in email log
if len(msg.recipients) == 1:
# parse "to" field; email address is returned as second member of a 2-tuple
pair = parseaddr(msg.recipients[0])
user = User.query.filter_by(email=pair[1]).first()
if user is not None:
log = EmailLog(user_id=user.id,
recipient=None,
send_date=datetime.now(),
subject=msg.subject,
body=msg.body,
html=msg.html)
if log is None:
log = EmailLog(user_id=None,
recipient=', '.join(msg.recipients),
send_date=datetime.now(),
subject=msg.subject,
body=msg.body,
html=msg.html)
db.session.add(log)
db.session.commit()
except SQLAlchemyError as e:
current_app.logger.exception("SQLAlchemyError exception", exc_info=e)
raise self.retry()
def dispatch_student_single_email(self, user_id, n_id):
try:
user = db.session.query(User).filter_by(id=user_id).first()
notification = db.session.query(EmailNotification).filter_by(
id=n_id).first()
except SQLAlchemyError as e:
current_app.logger.exception("SQLAlchemyError exception",
exc_info=e)
raise self.retry()
if user is None or notification is None:
self.update_state('FAILURE',
meta='Could not read database records')
raise Ignore()
msg = Message(sender=current_app.config['MAIL_DEFAULT_SENDER'],
reply_to=current_app.config['MAIL_REPLY_TO'],
recipients=[user.email],
subject=notification.msg_subject())
msg.body = render_template('email/notifications/student/single.txt',
user=user,
notification=notification)
# register a new task in the database
task_id = register_task(
msg.subject,
description='Send notification email to {r}'.format(
r=', '.join(msg.recipients)))
# queue Celery task to send the email
send_log_email = celery.tasks[
'app.tasks.send_log_email.send_log_email']
send_log_email.apply_async(args=(task_id, msg), task_id=task_id)
return n_id
def provider_do_work(self, action: ProviderAction, provider_pk: int,
model: str, model_pk, **kwargs):
"""Run the actual saving procedure of a single provider and keep trying on failure"""
self.prepare(**kwargs)
LOGGER.debug("Starting provider_do_work %r", action)
try:
self.progress.set(1)
count = 0
results = []
provider_object_generator = provider_resolve_helper(
provider_pk, model, model_pk)
for provider_object in provider_object_generator:
result = None
if action == ProviderAction.SAVE:
result = provider_object.save(**kwargs)
elif action == ProviderAction.DELETE:
result = provider_object.delete(**kwargs)
count += 1
results.append((provider_object.__class__.__name__, result))
self.progress.set(count)
LOGGER.debug("\tUpdated instance %r", provider_object)
self.progress.set(100)
return {provider_pk: results}
except ProviderRetryException as exc:
LOGGER.warning(exc)
raise self.retry(args=[action, provider_pk, model, model_pk],
kwargs=kwargs,
countdown=2**self.request.retries)
except SupervisrProviderException as exc:
# self.update_state(
# state=states.FAILURE,
# meta={'error': str(exc)})
# ignore the task so no other state is recorded
raise Ignore()
except Exception as exc: # pylint: disable=broad-except
handle_unexpected_error(exc, self.invoker)
def send_marking_emails(self, record_id, cc_convenor, max_attachment,
test_email, deadline, convenor_id):
try:
record: SubmissionPeriodRecord = db.session.query(
SubmissionPeriodRecord).filter_by(id=record_id).first()
except SQLAlchemyError as e:
current_app.logger.exception("SQLAlchemyError exception",
exc_info=e)
raise self.retry()
if record is None:
self.update_state(
'FAILURE',
meta='Could not load SubmissionPeriodRecord from database')
raise Ignore()
print(
'-- Send marking emails for project class "{proj}", submission period '
'"{period}"'.format(proj=record.config.name,
period=record.display_name))
print('-- configuration: CC convenor = {cc}, max attachment '
'total = {max} Mb'.format(cc=cc_convenor, max=max_attachment))
if test_email is not None:
print('-- working in test mode: emails being sent to sink={email}'.
format(email=test_email))
print('-- supplied deadline is {deadline}'.format(
deadline=parser.parse(deadline).date()))
email_group = group(
dispatch_emails.s(s.id, cc_convenor, max_attachment, test_email,
deadline)
for s in record.submissions) | notify_dispatch.s(convenor_id)
raise self.replace(email_group)
def monitor(self, arithmetic):
response = self.watcher.get_message()
if response:
channel = response['channel']
msg = response["data"]
if msg != 1:
logger.debug(f'接受到订阅信息:{channel} {msg}')
msg = json.loads(response["data"].decode(encoding='utf8'))
command = msg['command']
params = msg['params']
if CommandCode(command) == CommandCode.STOP:
logger.info(f'{self.request.id}: 接受到*终止*信息')
arithmetic.process.status = StatusCode.STOP
self.update_state(state=StatusCode.STOP)
raise Ignore()
elif CommandCode(command) == CommandCode.PAUSE:
logger.info(f'{self.request.id}: 接受到*暂停*信息')
arithmetic.process.status = StatusCode.PAUSE
elif CommandCode(command) == CommandCode.RESUME:
arithmetic.process.status = StatusCode.PROGRESS
logger.info(f'{self.request.id}: 接受到*恢复*信息')
else:
raise ValueError(f'无效的命令:{command}')
def precomputeOutliers(self, data: Any, combinations, record_id, project):
data = pd.read_json(data)
# self.update_state(state=STARTED)
combinations = list(filter(lambda x: len(x) < 3, combinations))
to_process = sum([get_dbscan_count(data[combo]) for combo in combinations])
processed = 0
self.update_state(state=STARTED,
meta={
"processed": processed,
"to_process": to_process
})
results = []
for combo in combinations:
subset = data[combo]
dimensions = ",".join(combo)
for result in computeDBScan(subset, dimensions, record_id, True):
results.append(result)
processed += 1
self.update_state(state=STARTED,
meta={
"processed": processed,
"to_process": to_process
})
with getSessionScopeFromId(project) as session:
session.add_all(results)
self.update_state(state=SUCCESS,
meta={
"processed": processed,
"to_process": to_process
})
raise Ignore()
def reset_batch(self, ident):
try:
record = db.session.query(StudentBatch).filter_by(id=ident).first()
except SQLAlchemyError as e:
current_app.logger.exception("SQLAlchemyError exception",
exc_info=e)
raise self.retry()
if record is None:
self.update_state('FAILURE', 'Could not read database records')
raise Ignore()
record.celery_finished = True
record.success = False
try:
db.session.commit()
except SQLAlchemyError as e:
db.session.rollback()
current_app.logger.exception("SQLAlchemyError exception",
exc_info=e)
raise self.retry()
return True
def email_notification(self, sent_data, user_id, message, priority):
try:
user = db.session.query(User).filter_by(id=user_id).first()
except SQLAlchemyError as e:
current_app.logger.exception("SQLAlchemyError exception",
exc_info=e)
raise self.retry()
if user is None:
self.update_state('FAILURE',
meta='Could not load User record from database')
raise Ignore()
if not isinstance(sent_data, list):
sent_data = [sent_data]
num_sent = sum([n for n in sent_data if n is not None])
plural = 's'
if num_sent == 1:
plural = ''
user.post_message(message.format(n=num_sent, pl=plural),
priority,
autocommit=True)
def set_notified(self, config_id, user_id):
try:
config: ProjectClassConfig = ProjectClassConfig.query.filter_by(
id=config_id).first()
user = User.query.filter_by(id=user_id).first()
except SQLAlchemyError as e:
current_app.logger.exception("SQLAlchemyError exception",
exc_info=e)
raise self.retry()
if user is None or config is None:
self.update_state('FAILURE',
meta='Could not load database records')
raise Ignore()
config.golive_notified.append(user)
try:
db.session.commit()
except SQLAlchemyError as e:
db.session.rollback()
current_app.logger.exception("SQLAlchemyError exception",
exc_info=e)
raise self.retry()
def matching_enumeration_record_maintenance(self, id):
try:
record = db.session.query(MatchingEnumeration).filter_by(
id=id).first()
except SQLAlchemyError as e:
current_app.logger.exception("SQLAlchemyError exception",
exc_info=e)
raise self.retry()
if record is None:
raise Ignore()
if not record.matching.awaiting_upload:
# can purge this record
try:
db.session.delete(record)
db.session.commit()
except SQLAlchemyError as e:
db.session.rollback()
current_app.logger.exception("SQLAlchemyError exception",
exc_info=e)
raise self.retry()
self.update_state(state='SUCCESS')
def verify_sequences_task(self, task_type, task_id, upload_paths_dict, seq_output_path, input_format):
current_process().daemon = False
current_process()._authkey = current_process().authkey
current_process()._daemonic = current_process().daemon
current_process()._tempdir = current_process()._config.get('tempdir')
if task_type == "inference":
status = inference_status
elif task_type == "prediction":
status = prediction_status
else:
raise ValueError
try:
verify_sequences_helper(task_id, upload_paths_dict, seq_output_path, input_format)
except ValueError, e:
if len(e.args) == 5:
status.send(
sender=self.__name__,
id=task_id,
success=False,
message=e.args[0],
linenum=e.args[1],
filetype=e.args[2],
culprit=e.args[3]
)
else:
status.send(
sender=self.__name__,
id=task_id,
success=False,
message="Error with input files"
)
self.update_state(
state = states.FAILURE,
)
raise Ignore()
def replace(self, sig):
"""Replace the current task, with a new task inheriting the
same task id.
:param sig: :class:`@signature`
Note: This will raise :exc:`~@Ignore`, so the best practice
is to always use ``raise self.replace(...)`` to convey
to the reader that the task will not continue after being replaced.
:param: Signature of new task.
"""
chord = self.request.chord
if isinstance(sig, group):
sig |= self.app.tasks['celery.accumulate'].s(index=0).set(
chord=chord, )
chord = None
sig.freeze(self.request.id,
group_id=self.request.group,
chord=chord,
root_id=self.request.root_id)
sig.delay()
raise Ignore('Chord member replaced by new task')
def bed_extract_task(self, task_type, task_id, bedtools_args_dicts, bedtools_path, genome, upload_paths_dict, input_format, seq_output_path, bed_output_path, output_control_path=None):
current_process().daemon = False
current_process()._authkey = current_process().authkey
current_process()._daemonic = current_process().daemon
current_process()._tempdir = current_process()._config.get('tempdir')
if task_type == "inference":
status = inference_status
elif task_type == "prediction":
status = prediction_status
else:
raise ValueError
try:
bed_setup_helper(task_id, genome, upload_paths_dict["bed"], input_format, bed_output_path, output_control_path):
except ValueError, e:
if len(e.args) == 5:
status.send(
sender=self.__name__,
id=task_id,
success=False,
message=e.args[0],
linenum=e.args[1],
filetype=e.args[2],
culprit=e.args[3]
)
else:
status.send(
sender=self.__name__,
id=task_id,
success=False,
message="Error with input BED files"
)
self.update_state(
state = states.FAILURE,
)
raise Ignore()
def ignored():
raise Ignore()
def run(self, preset_quality, sleep_time=5, *args, **kwargs):
"""Launch video transcoding.
For each of the presets generate a new ``ObjectVersion`` tagged as
slave with the preset name as key and a link to the master version.
:param self: reference to instance of task base class
:param preset_quality: preset quality to use for transcoding.
:param sleep_time: time interval between requests for the Sorenson
status.
"""
self._base_payload.update(preset_quality=preset_quality)
# Get master file's bucket_id
bucket_id = self.object.bucket_id
bucket_location = self.object.bucket.location.uri
# Get master file's key
master_key = self.object.key
tags = self.object.get_tags()
# Get master file's aspect ratio
aspect_ratio = tags['display_aspect_ratio']
# Get master file's width x height
width = int(tags['width']) if 'width' in tags else None
height = int(tags['height']) if 'height' in tags else None
with db.session.begin_nested():
# Create FileInstance
file_instance = FileInstance.create()
# Create ObjectVersion
obj_key = self._build_slave_key(preset_quality=preset_quality,
master_key=master_key)
obj = ObjectVersion.create(bucket=bucket_id, key=obj_key)
# Extract new location
storage = file_instance.storage(default_location=bucket_location)
directory, filename = storage._get_fs()
input_file = self.object.file.uri
output_file = os.path.join(directory.root_path, filename)
try:
# Start Sorenson
job_id = start_encoding(input_file,
output_file,
preset_quality,
aspect_ratio,
max_height=height,
max_width=width)
except (InvalidResolutionError, TooHighResolutionError) as e:
exception = self._meta_exception_envelope(exc=e)
self.update_state(state=REVOKED, meta=exception)
raise Ignore()
# Set revoke handler, in case of an abrupt execution halt.
self.set_revoke_handler(partial(stop_encoding, job_id))
# Create ObjectVersionTags
ObjectVersionTag.create(obj, 'master', self.obj_id)
ObjectVersionTag.create(obj, '_sorenson_job_id', job_id)
ObjectVersionTag.create(obj, 'preset_quality', preset_quality)
ObjectVersionTag.create(obj, 'media_type', 'video')
ObjectVersionTag.create(obj, 'context_type', 'subformat')
preset_info = get_preset_info(aspect_ratio, preset_quality)
for key, value in preset_info.items():
ObjectVersionTag.create(obj, key, value)
# Information necessary for monitoring
job_info = dict(
preset_quality=preset_quality,
job_id=job_id,
file_instance=str(file_instance.id),
uri=output_file,
version_id=str(obj.version_id),
key=obj_key,
tags=obj.get_tags(),
percentage=0,
)
db.session.commit()
self.update_state(state=STARTED,
meta=dict(payload=dict(**job_info),
message='Started transcoding.'))
status = ''
# Monitor job and report accordingly
while status != 'Finished':
# Get job status
status, percentage = get_encoding_status(job_id)
if status == 'Error':
raise RuntimeError('Error transcoding')
job_info['percentage'] = percentage
# Update task's state for this preset
self.update_state(
state=STARTED,
meta=dict(payload=dict(**job_info),
message='Transcoding {0}'.format(percentage)))
time.sleep(sleep_time)
# Set file's location, if job has completed
self._clean_file_name(output_file)
with db.session.begin_nested():
uri = output_file
with open(uri, 'rb') as transcoded_file:
digest = hashlib.md5(transcoded_file.read()).hexdigest()
size = os.path.getsize(uri)
checksum = '{0}:{1}'.format('md5', digest)
file_instance.set_uri(uri, size, checksum)
as_object_version(job_info['version_id']).set_file(file_instance)
db.session.commit()
def main(self, root_dir, root_directory, study, subject, dbusername,
dbpassword, dbhost, dbport, dbauthsource, dbname):
# Raise an exception if the import directory does not exist.
if not os.path.isdir(root_directory):
logger.error(
'{PATH} does not exist. Exiting.'.format(PATH=root_directory))
self.update_state(
state=states.FAILURE,
meta='{PATH} does not exist.'.format(PATH=root_directory))
raise Ignore()
# Celery status update
if subject and study:
self.update_state(state='PROCESSING {SUB} in {STUDY}'.format(
SUB=subject, STUDY=study))
else:
self.update_state(state='PROCESSING')
# Connect to the database
logger.info('connecting to database')
uri = 'mongodb://' + str(dbusername) + ':' + str(dbpassword) + '@' + str(
dbhost) + ':' + str(dbport) + '/' + str(dbauthsource)
client = MongoClient(
uri,
ssl=True,
ssl_certfile=configuration['mongodb']['ssl']['ssl_certfile'],
ssl_keyfile=configuration['mongodb']['ssl']['ssl_keyfile'],
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=configuration['mongodb']['ssl']['ca_certs'])
db = client[dbname]
# clean out incomplete table-of-content entries
logger.info('Sanitizing database')
dbtools.sanitize(db)
try:
for import_dir, dirs, files in os.walk(root_directory):
# Filter out hidden files and directories
files = [f for f in files if not f[0] == '.']
dirs[:] = [d for d in dirs if not d[0] == '.']
# Filter out directories for the raw data
dirs[:] = [d for d in dirs if not d == 'raw']
for file_name in files:
if len(file_name.split('-')) == 4 or file_name.endswith(
'_metadata.csv'):
import_file(import_dir, file_name, db)
except Exception as e:
logger.error(e)
if subject and study:
clean_toc_subject(db, study, subject)
elif study:
clean_toc_study(db, study)
else:
clean_toc(db)
max_days = get_lastday(db)
if max_days:
clean_metadata(db, max_days)
) self.update_state( state = states.FAILURE, ) raise Ignore() except Exception: status.send( sender=self.__name__, id=task_id, success=False, message="Error with input BED files" ) self.update_state( state = states.FAILURE, ) raise Ignore() bedtools_jobs = [ Process(target=run_bedtools_helper, args=(i, bedtools_path)) for i in bedtools_args_dicts ] for j in bedtools_jobs: j.start() for j in bedtools_jobs: j.join() for j in bedtools_jobs: try: j.get() except Exception: status.send( sender=self.__name__, id=task_id, success=False,
def raise_fail(task_id, message):
archive_item.update_state(task_id,
state=states.FAILURE,
meta={'error': message})
# by raising Ignore the processing will stop(no retries) and the result is not updated anymore
raise Ignore()
def get_time_series_data(self, fitbit_user, cat, resource, date=None):
""" Get the user's time series data """
try:
_type = TimeSeriesDataType.objects.get(category=cat, resource=resource)
except TimeSeriesDataType.DoesNotExist as e:
logger.exception("The resource %s in category %s doesn't exist" %
(resource, cat))
raise Reject(e, requeue=False)
# Create a lock so we don't try to run the same task multiple times
sdat = date.strftime('%Y-%m-%d') if date else 'ALL'
lock_id = '{0}-lock-{1}-{2}-{3}'.format(__name__, fitbit_user, _type, sdat)
if not cache.add(lock_id, 'true', LOCK_EXPIRE):
logger.debug('Already retrieving %s data for date %s, user %s' %
(_type, fitbit_user, sdat))
raise Ignore()
try:
fbusers = UserFitbit.objects.filter(fitbit_user=fitbit_user)
default_period = utils.get_setting('FITAPP_DEFAULT_PERIOD')
if default_period:
dates = {'base_date': 'today', 'period': default_period}
else:
dates = {'base_date': 'today', 'period': 'max'}
if date:
dates = {'base_date': date, 'end_date': date}
for fbuser in fbusers:
data = utils.get_fitbit_data(fbuser, _type, **dates)
if utils.get_setting('FITAPP_GET_INTRADAY'):
tz_offset = utils.get_fitbit_profile(fbuser,
'offsetFromUTCMillis')
tz_offset = tz_offset / 3600 / 1000 * -1 # Converted to positive hours
for datum in data:
# Create new record or update existing record
date = parser.parse(datum['dateTime'])
if _type.intraday_support and \
utils.get_setting('FITAPP_GET_INTRADAY'):
resources = TimeSeriesDataType.objects.filter(
intraday_support=True)
for i, _type in enumerate(resources):
# Offset each call by 2 seconds so they don't bog down
# the server
get_intraday_data(fbuser.fitbit_user, _type.category,
_type.resource, date, tz_offset)
tsd, created = TimeSeriesData.objects.get_or_create(
user=fbuser.user,
resource_type=_type,
date=date,
intraday=False)
tsd.value = datum['value']
tsd.save()
# Release the lock
cache.delete(lock_id)
except HTTPTooManyRequests as e:
# We have hit the rate limit for the user, retry when it's reset,
# according to the reply from the failing API call
countdown = e.retry_after_secs + int(
# Add exponential back-off + random jitter
random.uniform(2, 4)**self.request.retries)
logger.debug('Rate limit reached, will try again in {} seconds'.format(
countdown))
raise get_time_series_data.retry(exc=e, countdown=countdown)
except HTTPBadRequest as e:
# If the resource is elevation or floors, we are just getting this
# error because the data doesn't exist for this user, so we can ignore
# the error
if not ('elevation' in resource or 'floors' in resource):
exc = sys.exc_info()[1]
logger.exception("Exception updating data for user %s: %s" %
(fitbit_user, exc))
raise Reject(exc, requeue=False)
except Exception:
exc = sys.exc_info()[1]
logger.exception("Exception updating data for user %s: %s" %
(fitbit_user, exc))
raise Reject(exc, requeue=False)
def hashcatCrackMode(self, crackOptions, dicoOptions, dicoRulesOptions,
selectedKeywords, selectedMask, output_file, options):
# We wrap all the calls to the crack modes to catch the
# NoRemainingHashException exception
try:
# Keywords attack
if "Keywords" in crackOptions:
# Test the "pure" keywords
output_file_key = output_file + ":Keywords::"
KeywordsCrackMode(options).run(output_file=output_file_key,
keywords=selectedKeywords)
# Test variations of the keywords
for rule_file_name in conf.rule_name_list:
output_file_keyVar = output_file_key + rule_file_name
KeywordsCrackMode(options).run(output_file=output_file_keyVar,
keywords=selectedKeywords,
rule=rule_file_name)
# Wordlist attack
if "Wordlist" in crackOptions:
for wordlist in dicoOptions:
output_file_dict = output_file + ":Wordlist:" + wordlist + ":"
WordlistBasedCrackMode(options).run(
output_file=output_file_dict, wordlist=wordlist)
# Wordlist variations attack
if "WordlistVariations" in crackOptions:
for wordlist in dicoRulesOptions:
# For each wordlist to use with rules (eg
# "dictionnaire-francais")
for rule_name in conf.rule_name_list:
# For each rule (eg "rockyou-30000.rule")
output_file_dictVar = output_file + ":Wordlist:" + wordlist + ":" + rule_name
WordlistBasedCrackMode(options).run(
output_file=output_file_dictVar,
wordlist=wordlist,
rule=rule_name)
# Mask attack
if "Mask" in crackOptions:
output_file_mask = output_file + ":Mask:" + selectedMask + ":"
MaskCrackMode(options).run(output_file=output_file_mask,
mask=selectedMask)
# Brute force attack
if "Bruteforce" in crackOptions:
# Equal to a 20 chacracters mask
output_file_bf = output_file + ":Bruteforce::"
BruteforceCrackMode(options).run(output_file=output_file_bf)
except NoRemainingHashException:
print("Session finished early, we found all the hashes :)")
except RevokedTaskException:
print("Revoked/aborted task!")
# Mark it as revoked, for the list page
self.update_state(state='REVOKED')
# We need to raise this Celery exception, otherwise the worker will
# set the SUCCESS status after finishing (successfully) the task.
raise Ignore()
return True
def alter_priority(priority):
if priority is None:
raise Ignore()
priors = [p[0] for p in priorities]
return min(priors.index(priority) - 2, 1)
def run_program(self, data):
"""
The task to use to build/run/prove/submit a project. This gets queued up by a flask route.
:param self:
This is the reference to the celery task
:param data:
The json data from the flask route that contains the files and run mode
:return:
Returns a dict containing the status code from the execution
"""
try:
start = time.time()
container = DockerContainer(celery.conf['CONTAINER_NAME'])
task_id = self.request.id
mode = data['mode']
app = self._app
if mode == "run":
project = RemoteProject(app, container, task_id, data)
project.build()
code, out = project.run()
elif mode == "submit":
project = RemoteProject(app, container, task_id, data)
project.build()
project.submit()
code = 0
elif mode == "compile":
project = RemoteProject(app, container, task_id, data)
code = project.build()
elif mode == "prove":
project = RemoteProject(app, container, task_id, data, True)
code = project.prove([])
elif mode == "prove_flow":
project = RemoteProject(app, container, task_id, data, True)
code = project.prove(["--mode=flow"])
elif mode == "prove_flow_report_all":
project = RemoteProject(app, container, task_id, data, True)
code = project.prove(["--mode=flow", "--report=all"])
elif mode == "prove_report_all":
project = RemoteProject(app, container, task_id, data, True)
code = project.prove(["--report=all"])
else:
raise Exception(f"Mode {mode} not implemented")
except BuildError as ex:
# Build errors can be common - syntax errors and such
# Use logger.debug here to minimize unwanted traffic in celery logs
logger.debug(f"Build error code {ex}", exc_info=True)
elapsed = time.time() - start
return {'status': int(f"{ex}"), 'elapsed': elapsed}
except Exception as ex:
logger.error("An error occured executing the command", exc_info=True)
self.update_state(state=states.FAILURE,
meta={
'exc_type': type(ex).__name__,
'exc_message': traceback.format_exc().split('\n')
})
raise Ignore()
finally:
if container:
container.remove()
elapsed = time.time() - start
return {'status': code, 'elapsed': elapsed}
def start_shoalscrape_task(shoalscrape_task_id, fail_silently=False):
# Reference: http://stackoverflow.com/a/8096086
# Allows us to terminate this task's execution arbitrarily through Django.
# This section should come as early as possible.
current_task_id = start_shoalscrape_task.request.id
already_running_tasks = ShoalScrapeTask.objects.filter(
id=shoalscrape_task_id).exclude(current_task_id='')
# Checking same_db_id_and_different_task_id does block multiple workers.
if already_running_tasks.exists():
other_celery_ids = [t.current_task_id for t in already_running_tasks]
shoalscrape_ids = [t.id for t in already_running_tasks]
error_message = ('[ ! ] ShoalScrapeTask(s) {} attempted to start a'
' Celery worker with ID {} while they already had'
' Celery workers with IDs {}. Terminating worker with'
' Celery task ID {}'.format(shoalscrape_ids,
current_task_id,
other_celery_ids,
current_task_id))
revoke(current_task_id, terminate=True)
logger.info(error_message)
raise Ignore()
try:
shoalscrape_task = ShoalScrapeTask.objects.get(id=shoalscrape_task_id)
except ShoalScrapeTask.DoesNotExist:
# Prevents Celery log spam for the defunct task. (different from VE)
ptask_identifier = 'shoalscrape_task_{}'.format(shoalscrape_task_id)
ptask = PeriodicTask.objects.get(name=ptask_identifier)
task_error_log_template = ('ShoalScrapeTask #{} not found while '
'starting. PeriodicTask #{}, {}, has been '
'deleted.')
logger.info(
task_error_log_template.format(shoalscrape_task_id, ptask.id,
ptask.name))
ptask.delete()
raise Ignore()
# This should come as early as possible after checking that the task is not
# duplicated (ie, once current_task_id and shoalscrape_task are available):
shoalscrape_task.current_task_id = current_task_id
shoalscrape_task.save()
ptask = shoalscrape_task.periodic_task
# Prevents any other Celery workers from starting a task with the same ID:
ptask.last_run_at = datetime.datetime.now() + datetime.timedelta(days=9999)
ptask.save()
shoalscrape_task.save()
active_tasks = ShoalScrapeTask.objects.filter(
Q(periodic_task__enabled=True)
| Q(state=ShoalScrapeTask.IN_PROGRESS)).exclude(id=shoalscrape_task_id)
# Checking active_tasks does not protect against multiple Celery workers
# starting for the same ShoalScrapeTask; rather, it prevents multiple
# ShoalScrapeTasks from starting.
if active_tasks.exists():
ids = [each.id for each in active_tasks]
error_message = ('[ ! ] Started while other ShoalScrape tasks were '
'running: {}'.format(ids))
logger.info(error_message)
shoalscrape_task.error = error_message
shoalscrape_task.save()
shoalscrape_task.set_state(ShoalScrapeTask.ERROR)
shoalscrape_task.terminate_task(log_info=error_message)
raise Ignore()
server_tz = dj_tz.get_default_timezone()
server_now = server_tz.localize(datetime.datetime.now())
shoalscrape_task.last_started_at = server_now
shoalscrape_task.save()
task_log_template = '[ + ] Attempting to start ShoalScrapeTask #{} at {}'
logger.info(
task_log_template.format(shoalscrape_task.id,
shoalscrape_task.last_started_at))
results = 0
try:
company = shoalscrape_task.company
domain = shoalscrape_task.domain
company_linkedin_id = shoalscrape_task.company_linkedin_id
log_file_path = shoalscrape_task.path
creds = shoalscrape_task.shoalscrape_creds
user_agent = None
if creds.scraper_user_agent:
user_agent = creds.scraper_user_agent.user_agent_data
username = creds.username
password = creds.password
results = shoalscrape.webserver_main(company,
company_linkedin_id,
domain,
log_file_path,
user_agent,
username=username,
password=password,
scan_profiles=False,
brute_force_emails=False)
shoalscrape_task.set_state(ShoalScrapeTask.COMPLETE)
shoalscrape_task.current_task_id = ''
shoalscrape_task.save()
except Exception as e:
error = ('[ ! ] ShoalScrapeTask'
' #{} had an error: {}'.format(shoalscrape_task.id, e))
shoalscrape_task.set_state(ShoalScrapeTask.ERROR)
shoalscrape_task.terminate_task(log_info=error)
shoalscrape_task.error = error
shoalscrape_task.save()
logger.info(error)
raise Ignore()
return results
