def email_failure(self, subject, user_id): try: record: User = db.session.query(User).filter_by(id=user_id).first() except SQLAlchemyError as e: current_app.logger.exception("SQLAlchemyError exception", exc_info=e) raise self.retry() if record is None: self.update_state('FAILURE', meta='Could not load database records') raise Ignore() record.post_message( 'An error occurred and your email with subject "{subj}" was not sent ' 'to all recipients. Please check the email log to determine which instances were ' 'sent correctly. You may wish to consult with a system ' 'administrator.'.format(subj=subject), 'error', autocommit=True) return True
def replace(self, sig): """Replace this task, with a new task inheriting the task id. .. versionadded:: 4.0 Arguments: sig (~@Signature): signature to replace with. Raises: ~@Ignore: This is always raised, so the best practice is to always use ``raise self.replace(...)`` to convey to the reader that the task won't continue after being replaced. """ chord = self.request.chord if 'chord' in sig.options: if chord: chord = sig.options['chord'] | chord else: chord = sig.options['chord'] if isinstance(sig, group): sig |= self.app.tasks['celery.accumulate'].s(index=0).set( chord=chord, link=self.request.callbacks, link_error=self.request.errbacks, ) chord = None if self.request.chain: for t in self.request.chain: sig |= signature(t, app=self.app) sig.freeze(self.request.id, group_id=self.request.group, chord=chord, root_id=self.request.root_id) sig.delay() raise Ignore('Replaced by new task')
def replace(self, sig): """Replace the current task, with a new task inheriting the same task id. :param sig: :class:`@signature` .. versionadded:: 4.0 Note: This will raise :exc:`~@Ignore`, so the best practice is to always use ``raise self.replace(...)`` to convey to the reader that the task will not continue after being replaced. """ chord = self.request.chord if 'chord' in sig.options: if chord: chord = sig.options['chord'] | chord else: chord = sig.options['chord'] if isinstance(sig, group): sig |= self.app.tasks['celery.accumulate'].s(index=0).set( chord=chord, link=self.request.callbacks, link_error=self.request.errbacks, ) chord = None if self.request.chain: for t in self.request.chain: sig |= signature(t) sig.freeze(self.request.id, group_id=self.request.group, chord=chord, root_id=self.request.root_id) sig.delay() raise Ignore('Replaced by new task')
def precomputeLR(self, data: Any, combinations, record_id, project): data = pd.read_json(data) # TODO: For higher dimensions do we iterate over whi. h will be the dependant variable? combinations = list(filter(lambda x: len(x) == 2, combinations)) to_process = 2 * len(combinations) processed = 0 self.update_state(state=STARTED, meta={ "processed": processed, "to_process": to_process }) results = [] for combo in combinations: subset = data[combo] dimensions = ",".join(combo) for result in computeLR(subset, dimensions, record_id): results.append(result) processed += 1 self.update_state(state=STARTED, meta={ "processed": processed, "to_process": to_process }) with getSessionScopeFromId(project) as session: session.add_all(results) self.update_state(state=SUCCESS, meta={ "processed": processed, "to_process": to_process }) raise Ignore()
def run(self): idle_drives = TapeDrive.objects.filter( status=20, storage_medium__isnull=False, last_change__lte=timezone.now() - F('idle_time'), locked=False, ) if not idle_drives.exists(): raise Ignore() for drive in idle_drives.iterator(): robot_queue_entry_exists = RobotQueue.objects.filter( storage_medium=drive.storage_medium, req_type=20, status__in=[0, 2]).exists() if not robot_queue_entry_exists: RobotQueue.objects.create( user=User.objects.get(username='******'), storage_medium=drive.storage_medium, req_type=20, status=0, )
def long_running_task(self, name): redis_key = "celery-locks-{0}".format(name) lock = redis_store.lock(redis_key) has_lock = False try: has_lock = lock.acquire(blocking=False) if has_lock: label = "Processing {0}".format(name) self.update_state(state=u"IN PROGRESS", meta=label) x = 10 while x > 0: x = x - 1 time.sleep(1) else: label = "Task {0} is processed by another worker.".format(name) self.update_state(state=states.FAILURE, meta=label) raise Ignore() finally: if has_lock: lock.release() return "Finished with {0}. Phew!".format(name)
def book_scraper_chaps_update(self, s_from=0, s_to=0): """ Create update task for each novel with revisit=True """ books = Book.objects.filter(visited=True).exclude(visit_id__iexact='') interval = 15 for book in books: if book.chapters_count and book.revisit_id and not book.revisited: try: interval += 3 book.revisited = True book.save() schedule, created = IntervalSchedule.objects.get_or_create( every=interval, period=IntervalSchedule.SECONDS, ) PeriodicTask.objects.create( one_off=True, interval=schedule, name=f'Update book chapters: {book.title}', task='novel2read.apps.books.tasks.book_revisit_novel', args=json.dumps([book.pk]), kwargs=json.dumps({ 's_from': s_from, 's_to': s_to, }), ) except Exception as exc: exc_result = '\n'.join([f'Book: {book.title}', f'{exc}']) save_celery_result( task_id=self.request.id, task_name=self.name, status=states.FAILURE, result=exc_result, traceback=traceback.format_exc(), ) raise Ignore()
def log_email(self, task_id, msg: Message): progress_update(task_id, TaskRecord.RUNNING, 80, "Logging email in database...", autocommit=True) # don't log if we are not on a live email platform if not current_app.config.get('EMAIL_IS_LIVE', False): raise Ignore() try: log = None # store message in email log if len(msg.recipients) == 1: # parse "to" field; email address is returned as second member of a 2-tuple pair = parseaddr(msg.recipients[0]) user = User.query.filter_by(email=pair[1]).first() if user is not None: log = EmailLog(user_id=user.id, recipient=None, send_date=datetime.now(), subject=msg.subject, body=msg.body, html=msg.html) if log is None: log = EmailLog(user_id=None, recipient=', '.join(msg.recipients), send_date=datetime.now(), subject=msg.subject, body=msg.body, html=msg.html) db.session.add(log) db.session.commit() except SQLAlchemyError as e: current_app.logger.exception("SQLAlchemyError exception", exc_info=e) raise self.retry()
def dispatch_student_single_email(self, user_id, n_id): try: user = db.session.query(User).filter_by(id=user_id).first() notification = db.session.query(EmailNotification).filter_by( id=n_id).first() except SQLAlchemyError as e: current_app.logger.exception("SQLAlchemyError exception", exc_info=e) raise self.retry() if user is None or notification is None: self.update_state('FAILURE', meta='Could not read database records') raise Ignore() msg = Message(sender=current_app.config['MAIL_DEFAULT_SENDER'], reply_to=current_app.config['MAIL_REPLY_TO'], recipients=[user.email], subject=notification.msg_subject()) msg.body = render_template('email/notifications/student/single.txt', user=user, notification=notification) # register a new task in the database task_id = register_task( msg.subject, description='Send notification email to {r}'.format( r=', '.join(msg.recipients))) # queue Celery task to send the email send_log_email = celery.tasks[ 'app.tasks.send_log_email.send_log_email'] send_log_email.apply_async(args=(task_id, msg), task_id=task_id) return n_id
def provider_do_work(self, action: ProviderAction, provider_pk: int, model: str, model_pk, **kwargs): """Run the actual saving procedure of a single provider and keep trying on failure""" self.prepare(**kwargs) LOGGER.debug("Starting provider_do_work %r", action) try: self.progress.set(1) count = 0 results = [] provider_object_generator = provider_resolve_helper( provider_pk, model, model_pk) for provider_object in provider_object_generator: result = None if action == ProviderAction.SAVE: result = provider_object.save(**kwargs) elif action == ProviderAction.DELETE: result = provider_object.delete(**kwargs) count += 1 results.append((provider_object.__class__.__name__, result)) self.progress.set(count) LOGGER.debug("\tUpdated instance %r", provider_object) self.progress.set(100) return {provider_pk: results} except ProviderRetryException as exc: LOGGER.warning(exc) raise self.retry(args=[action, provider_pk, model, model_pk], kwargs=kwargs, countdown=2**self.request.retries) except SupervisrProviderException as exc: # self.update_state( # state=states.FAILURE, # meta={'error': str(exc)}) # ignore the task so no other state is recorded raise Ignore() except Exception as exc: # pylint: disable=broad-except handle_unexpected_error(exc, self.invoker)
def send_marking_emails(self, record_id, cc_convenor, max_attachment, test_email, deadline, convenor_id): try: record: SubmissionPeriodRecord = db.session.query( SubmissionPeriodRecord).filter_by(id=record_id).first() except SQLAlchemyError as e: current_app.logger.exception("SQLAlchemyError exception", exc_info=e) raise self.retry() if record is None: self.update_state( 'FAILURE', meta='Could not load SubmissionPeriodRecord from database') raise Ignore() print( '-- Send marking emails for project class "{proj}", submission period ' '"{period}"'.format(proj=record.config.name, period=record.display_name)) print('-- configuration: CC convenor = {cc}, max attachment ' 'total = {max} Mb'.format(cc=cc_convenor, max=max_attachment)) if test_email is not None: print('-- working in test mode: emails being sent to sink={email}'. format(email=test_email)) print('-- supplied deadline is {deadline}'.format( deadline=parser.parse(deadline).date())) email_group = group( dispatch_emails.s(s.id, cc_convenor, max_attachment, test_email, deadline) for s in record.submissions) | notify_dispatch.s(convenor_id) raise self.replace(email_group)
def monitor(self, arithmetic): response = self.watcher.get_message() if response: channel = response['channel'] msg = response["data"] if msg != 1: logger.debug(f'接受到订阅信息:{channel} {msg}') msg = json.loads(response["data"].decode(encoding='utf8')) command = msg['command'] params = msg['params'] if CommandCode(command) == CommandCode.STOP: logger.info(f'{self.request.id}: 接受到*终止*信息') arithmetic.process.status = StatusCode.STOP self.update_state(state=StatusCode.STOP) raise Ignore() elif CommandCode(command) == CommandCode.PAUSE: logger.info(f'{self.request.id}: 接受到*暂停*信息') arithmetic.process.status = StatusCode.PAUSE elif CommandCode(command) == CommandCode.RESUME: arithmetic.process.status = StatusCode.PROGRESS logger.info(f'{self.request.id}: 接受到*恢复*信息') else: raise ValueError(f'无效的命令:{command}')
def precomputeOutliers(self, data: Any, combinations, record_id, project): data = pd.read_json(data) # self.update_state(state=STARTED) combinations = list(filter(lambda x: len(x) < 3, combinations)) to_process = sum([get_dbscan_count(data[combo]) for combo in combinations]) processed = 0 self.update_state(state=STARTED, meta={ "processed": processed, "to_process": to_process }) results = [] for combo in combinations: subset = data[combo] dimensions = ",".join(combo) for result in computeDBScan(subset, dimensions, record_id, True): results.append(result) processed += 1 self.update_state(state=STARTED, meta={ "processed": processed, "to_process": to_process }) with getSessionScopeFromId(project) as session: session.add_all(results) self.update_state(state=SUCCESS, meta={ "processed": processed, "to_process": to_process }) raise Ignore()
def reset_batch(self, ident): try: record = db.session.query(StudentBatch).filter_by(id=ident).first() except SQLAlchemyError as e: current_app.logger.exception("SQLAlchemyError exception", exc_info=e) raise self.retry() if record is None: self.update_state('FAILURE', 'Could not read database records') raise Ignore() record.celery_finished = True record.success = False try: db.session.commit() except SQLAlchemyError as e: db.session.rollback() current_app.logger.exception("SQLAlchemyError exception", exc_info=e) raise self.retry() return True
def email_notification(self, sent_data, user_id, message, priority): try: user = db.session.query(User).filter_by(id=user_id).first() except SQLAlchemyError as e: current_app.logger.exception("SQLAlchemyError exception", exc_info=e) raise self.retry() if user is None: self.update_state('FAILURE', meta='Could not load User record from database') raise Ignore() if not isinstance(sent_data, list): sent_data = [sent_data] num_sent = sum([n for n in sent_data if n is not None]) plural = 's' if num_sent == 1: plural = '' user.post_message(message.format(n=num_sent, pl=plural), priority, autocommit=True)
def set_notified(self, config_id, user_id): try: config: ProjectClassConfig = ProjectClassConfig.query.filter_by( id=config_id).first() user = User.query.filter_by(id=user_id).first() except SQLAlchemyError as e: current_app.logger.exception("SQLAlchemyError exception", exc_info=e) raise self.retry() if user is None or config is None: self.update_state('FAILURE', meta='Could not load database records') raise Ignore() config.golive_notified.append(user) try: db.session.commit() except SQLAlchemyError as e: db.session.rollback() current_app.logger.exception("SQLAlchemyError exception", exc_info=e) raise self.retry()
def matching_enumeration_record_maintenance(self, id): try: record = db.session.query(MatchingEnumeration).filter_by( id=id).first() except SQLAlchemyError as e: current_app.logger.exception("SQLAlchemyError exception", exc_info=e) raise self.retry() if record is None: raise Ignore() if not record.matching.awaiting_upload: # can purge this record try: db.session.delete(record) db.session.commit() except SQLAlchemyError as e: db.session.rollback() current_app.logger.exception("SQLAlchemyError exception", exc_info=e) raise self.retry() self.update_state(state='SUCCESS')
def verify_sequences_task(self, task_type, task_id, upload_paths_dict, seq_output_path, input_format): current_process().daemon = False current_process()._authkey = current_process().authkey current_process()._daemonic = current_process().daemon current_process()._tempdir = current_process()._config.get('tempdir') if task_type == "inference": status = inference_status elif task_type == "prediction": status = prediction_status else: raise ValueError try: verify_sequences_helper(task_id, upload_paths_dict, seq_output_path, input_format) except ValueError, e: if len(e.args) == 5: status.send( sender=self.__name__, id=task_id, success=False, message=e.args[0], linenum=e.args[1], filetype=e.args[2], culprit=e.args[3] ) else: status.send( sender=self.__name__, id=task_id, success=False, message="Error with input files" ) self.update_state( state = states.FAILURE, ) raise Ignore()
def replace(self, sig): """Replace the current task, with a new task inheriting the same task id. :param sig: :class:`@signature` Note: This will raise :exc:`~@Ignore`, so the best practice is to always use ``raise self.replace(...)`` to convey to the reader that the task will not continue after being replaced. :param: Signature of new task. """ chord = self.request.chord if isinstance(sig, group): sig |= self.app.tasks['celery.accumulate'].s(index=0).set( chord=chord, ) chord = None sig.freeze(self.request.id, group_id=self.request.group, chord=chord, root_id=self.request.root_id) sig.delay() raise Ignore('Chord member replaced by new task')
def bed_extract_task(self, task_type, task_id, bedtools_args_dicts, bedtools_path, genome, upload_paths_dict, input_format, seq_output_path, bed_output_path, output_control_path=None): current_process().daemon = False current_process()._authkey = current_process().authkey current_process()._daemonic = current_process().daemon current_process()._tempdir = current_process()._config.get('tempdir') if task_type == "inference": status = inference_status elif task_type == "prediction": status = prediction_status else: raise ValueError try: bed_setup_helper(task_id, genome, upload_paths_dict["bed"], input_format, bed_output_path, output_control_path): except ValueError, e: if len(e.args) == 5: status.send( sender=self.__name__, id=task_id, success=False, message=e.args[0], linenum=e.args[1], filetype=e.args[2], culprit=e.args[3] ) else: status.send( sender=self.__name__, id=task_id, success=False, message="Error with input BED files" ) self.update_state( state = states.FAILURE, ) raise Ignore()
def ignored(): raise Ignore()
def run(self, preset_quality, sleep_time=5, *args, **kwargs): """Launch video transcoding. For each of the presets generate a new ``ObjectVersion`` tagged as slave with the preset name as key and a link to the master version. :param self: reference to instance of task base class :param preset_quality: preset quality to use for transcoding. :param sleep_time: time interval between requests for the Sorenson status. """ self._base_payload.update(preset_quality=preset_quality) # Get master file's bucket_id bucket_id = self.object.bucket_id bucket_location = self.object.bucket.location.uri # Get master file's key master_key = self.object.key tags = self.object.get_tags() # Get master file's aspect ratio aspect_ratio = tags['display_aspect_ratio'] # Get master file's width x height width = int(tags['width']) if 'width' in tags else None height = int(tags['height']) if 'height' in tags else None with db.session.begin_nested(): # Create FileInstance file_instance = FileInstance.create() # Create ObjectVersion obj_key = self._build_slave_key(preset_quality=preset_quality, master_key=master_key) obj = ObjectVersion.create(bucket=bucket_id, key=obj_key) # Extract new location storage = file_instance.storage(default_location=bucket_location) directory, filename = storage._get_fs() input_file = self.object.file.uri output_file = os.path.join(directory.root_path, filename) try: # Start Sorenson job_id = start_encoding(input_file, output_file, preset_quality, aspect_ratio, max_height=height, max_width=width) except (InvalidResolutionError, TooHighResolutionError) as e: exception = self._meta_exception_envelope(exc=e) self.update_state(state=REVOKED, meta=exception) raise Ignore() # Set revoke handler, in case of an abrupt execution halt. self.set_revoke_handler(partial(stop_encoding, job_id)) # Create ObjectVersionTags ObjectVersionTag.create(obj, 'master', self.obj_id) ObjectVersionTag.create(obj, '_sorenson_job_id', job_id) ObjectVersionTag.create(obj, 'preset_quality', preset_quality) ObjectVersionTag.create(obj, 'media_type', 'video') ObjectVersionTag.create(obj, 'context_type', 'subformat') preset_info = get_preset_info(aspect_ratio, preset_quality) for key, value in preset_info.items(): ObjectVersionTag.create(obj, key, value) # Information necessary for monitoring job_info = dict( preset_quality=preset_quality, job_id=job_id, file_instance=str(file_instance.id), uri=output_file, version_id=str(obj.version_id), key=obj_key, tags=obj.get_tags(), percentage=0, ) db.session.commit() self.update_state(state=STARTED, meta=dict(payload=dict(**job_info), message='Started transcoding.')) status = '' # Monitor job and report accordingly while status != 'Finished': # Get job status status, percentage = get_encoding_status(job_id) if status == 'Error': raise RuntimeError('Error transcoding') job_info['percentage'] = percentage # Update task's state for this preset self.update_state( state=STARTED, meta=dict(payload=dict(**job_info), message='Transcoding {0}'.format(percentage))) time.sleep(sleep_time) # Set file's location, if job has completed self._clean_file_name(output_file) with db.session.begin_nested(): uri = output_file with open(uri, 'rb') as transcoded_file: digest = hashlib.md5(transcoded_file.read()).hexdigest() size = os.path.getsize(uri) checksum = '{0}:{1}'.format('md5', digest) file_instance.set_uri(uri, size, checksum) as_object_version(job_info['version_id']).set_file(file_instance) db.session.commit()
def main(self, root_dir, root_directory, study, subject, dbusername, dbpassword, dbhost, dbport, dbauthsource, dbname): # Raise an exception if the import directory does not exist. if not os.path.isdir(root_directory): logger.error( '{PATH} does not exist. Exiting.'.format(PATH=root_directory)) self.update_state( state=states.FAILURE, meta='{PATH} does not exist.'.format(PATH=root_directory)) raise Ignore() # Celery status update if subject and study: self.update_state(state='PROCESSING {SUB} in {STUDY}'.format( SUB=subject, STUDY=study)) else: self.update_state(state='PROCESSING') # Connect to the database logger.info('connecting to database') uri = 'mongodb://' + str(dbusername) + ':' + str(dbpassword) + '@' + str( dbhost) + ':' + str(dbport) + '/' + str(dbauthsource) client = MongoClient( uri, ssl=True, ssl_certfile=configuration['mongodb']['ssl']['ssl_certfile'], ssl_keyfile=configuration['mongodb']['ssl']['ssl_keyfile'], ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=configuration['mongodb']['ssl']['ca_certs']) db = client[dbname] # clean out incomplete table-of-content entries logger.info('Sanitizing database') dbtools.sanitize(db) try: for import_dir, dirs, files in os.walk(root_directory): # Filter out hidden files and directories files = [f for f in files if not f[0] == '.'] dirs[:] = [d for d in dirs if not d[0] == '.'] # Filter out directories for the raw data dirs[:] = [d for d in dirs if not d == 'raw'] for file_name in files: if len(file_name.split('-')) == 4 or file_name.endswith( '_metadata.csv'): import_file(import_dir, file_name, db) except Exception as e: logger.error(e) if subject and study: clean_toc_subject(db, study, subject) elif study: clean_toc_study(db, study) else: clean_toc(db) max_days = get_lastday(db) if max_days: clean_metadata(db, max_days)
) self.update_state( state = states.FAILURE, ) raise Ignore() except Exception: status.send( sender=self.__name__, id=task_id, success=False, message="Error with input BED files" ) self.update_state( state = states.FAILURE, ) raise Ignore() bedtools_jobs = [ Process(target=run_bedtools_helper, args=(i, bedtools_path)) for i in bedtools_args_dicts ] for j in bedtools_jobs: j.start() for j in bedtools_jobs: j.join() for j in bedtools_jobs: try: j.get() except Exception: status.send( sender=self.__name__, id=task_id, success=False,
def raise_fail(task_id, message): archive_item.update_state(task_id, state=states.FAILURE, meta={'error': message}) # by raising Ignore the processing will stop(no retries) and the result is not updated anymore raise Ignore()
def get_time_series_data(self, fitbit_user, cat, resource, date=None): """ Get the user's time series data """ try: _type = TimeSeriesDataType.objects.get(category=cat, resource=resource) except TimeSeriesDataType.DoesNotExist as e: logger.exception("The resource %s in category %s doesn't exist" % (resource, cat)) raise Reject(e, requeue=False) # Create a lock so we don't try to run the same task multiple times sdat = date.strftime('%Y-%m-%d') if date else 'ALL' lock_id = '{0}-lock-{1}-{2}-{3}'.format(__name__, fitbit_user, _type, sdat) if not cache.add(lock_id, 'true', LOCK_EXPIRE): logger.debug('Already retrieving %s data for date %s, user %s' % (_type, fitbit_user, sdat)) raise Ignore() try: fbusers = UserFitbit.objects.filter(fitbit_user=fitbit_user) default_period = utils.get_setting('FITAPP_DEFAULT_PERIOD') if default_period: dates = {'base_date': 'today', 'period': default_period} else: dates = {'base_date': 'today', 'period': 'max'} if date: dates = {'base_date': date, 'end_date': date} for fbuser in fbusers: data = utils.get_fitbit_data(fbuser, _type, **dates) if utils.get_setting('FITAPP_GET_INTRADAY'): tz_offset = utils.get_fitbit_profile(fbuser, 'offsetFromUTCMillis') tz_offset = tz_offset / 3600 / 1000 * -1 # Converted to positive hours for datum in data: # Create new record or update existing record date = parser.parse(datum['dateTime']) if _type.intraday_support and \ utils.get_setting('FITAPP_GET_INTRADAY'): resources = TimeSeriesDataType.objects.filter( intraday_support=True) for i, _type in enumerate(resources): # Offset each call by 2 seconds so they don't bog down # the server get_intraday_data(fbuser.fitbit_user, _type.category, _type.resource, date, tz_offset) tsd, created = TimeSeriesData.objects.get_or_create( user=fbuser.user, resource_type=_type, date=date, intraday=False) tsd.value = datum['value'] tsd.save() # Release the lock cache.delete(lock_id) except HTTPTooManyRequests as e: # We have hit the rate limit for the user, retry when it's reset, # according to the reply from the failing API call countdown = e.retry_after_secs + int( # Add exponential back-off + random jitter random.uniform(2, 4)**self.request.retries) logger.debug('Rate limit reached, will try again in {} seconds'.format( countdown)) raise get_time_series_data.retry(exc=e, countdown=countdown) except HTTPBadRequest as e: # If the resource is elevation or floors, we are just getting this # error because the data doesn't exist for this user, so we can ignore # the error if not ('elevation' in resource or 'floors' in resource): exc = sys.exc_info()[1] logger.exception("Exception updating data for user %s: %s" % (fitbit_user, exc)) raise Reject(exc, requeue=False) except Exception: exc = sys.exc_info()[1] logger.exception("Exception updating data for user %s: %s" % (fitbit_user, exc)) raise Reject(exc, requeue=False)
def hashcatCrackMode(self, crackOptions, dicoOptions, dicoRulesOptions, selectedKeywords, selectedMask, output_file, options): # We wrap all the calls to the crack modes to catch the # NoRemainingHashException exception try: # Keywords attack if "Keywords" in crackOptions: # Test the "pure" keywords output_file_key = output_file + ":Keywords::" KeywordsCrackMode(options).run(output_file=output_file_key, keywords=selectedKeywords) # Test variations of the keywords for rule_file_name in conf.rule_name_list: output_file_keyVar = output_file_key + rule_file_name KeywordsCrackMode(options).run(output_file=output_file_keyVar, keywords=selectedKeywords, rule=rule_file_name) # Wordlist attack if "Wordlist" in crackOptions: for wordlist in dicoOptions: output_file_dict = output_file + ":Wordlist:" + wordlist + ":" WordlistBasedCrackMode(options).run( output_file=output_file_dict, wordlist=wordlist) # Wordlist variations attack if "WordlistVariations" in crackOptions: for wordlist in dicoRulesOptions: # For each wordlist to use with rules (eg # "dictionnaire-francais") for rule_name in conf.rule_name_list: # For each rule (eg "rockyou-30000.rule") output_file_dictVar = output_file + ":Wordlist:" + wordlist + ":" + rule_name WordlistBasedCrackMode(options).run( output_file=output_file_dictVar, wordlist=wordlist, rule=rule_name) # Mask attack if "Mask" in crackOptions: output_file_mask = output_file + ":Mask:" + selectedMask + ":" MaskCrackMode(options).run(output_file=output_file_mask, mask=selectedMask) # Brute force attack if "Bruteforce" in crackOptions: # Equal to a 20 chacracters mask output_file_bf = output_file + ":Bruteforce::" BruteforceCrackMode(options).run(output_file=output_file_bf) except NoRemainingHashException: print("Session finished early, we found all the hashes :)") except RevokedTaskException: print("Revoked/aborted task!") # Mark it as revoked, for the list page self.update_state(state='REVOKED') # We need to raise this Celery exception, otherwise the worker will # set the SUCCESS status after finishing (successfully) the task. raise Ignore() return True
def alter_priority(priority): if priority is None: raise Ignore() priors = [p[0] for p in priorities] return min(priors.index(priority) - 2, 1)
def run_program(self, data): """ The task to use to build/run/prove/submit a project. This gets queued up by a flask route. :param self: This is the reference to the celery task :param data: The json data from the flask route that contains the files and run mode :return: Returns a dict containing the status code from the execution """ try: start = time.time() container = DockerContainer(celery.conf['CONTAINER_NAME']) task_id = self.request.id mode = data['mode'] app = self._app if mode == "run": project = RemoteProject(app, container, task_id, data) project.build() code, out = project.run() elif mode == "submit": project = RemoteProject(app, container, task_id, data) project.build() project.submit() code = 0 elif mode == "compile": project = RemoteProject(app, container, task_id, data) code = project.build() elif mode == "prove": project = RemoteProject(app, container, task_id, data, True) code = project.prove([]) elif mode == "prove_flow": project = RemoteProject(app, container, task_id, data, True) code = project.prove(["--mode=flow"]) elif mode == "prove_flow_report_all": project = RemoteProject(app, container, task_id, data, True) code = project.prove(["--mode=flow", "--report=all"]) elif mode == "prove_report_all": project = RemoteProject(app, container, task_id, data, True) code = project.prove(["--report=all"]) else: raise Exception(f"Mode {mode} not implemented") except BuildError as ex: # Build errors can be common - syntax errors and such # Use logger.debug here to minimize unwanted traffic in celery logs logger.debug(f"Build error code {ex}", exc_info=True) elapsed = time.time() - start return {'status': int(f"{ex}"), 'elapsed': elapsed} except Exception as ex: logger.error("An error occured executing the command", exc_info=True) self.update_state(state=states.FAILURE, meta={ 'exc_type': type(ex).__name__, 'exc_message': traceback.format_exc().split('\n') }) raise Ignore() finally: if container: container.remove() elapsed = time.time() - start return {'status': code, 'elapsed': elapsed}
def start_shoalscrape_task(shoalscrape_task_id, fail_silently=False): # Reference: http://stackoverflow.com/a/8096086 # Allows us to terminate this task's execution arbitrarily through Django. # This section should come as early as possible. current_task_id = start_shoalscrape_task.request.id already_running_tasks = ShoalScrapeTask.objects.filter( id=shoalscrape_task_id).exclude(current_task_id='') # Checking same_db_id_and_different_task_id does block multiple workers. if already_running_tasks.exists(): other_celery_ids = [t.current_task_id for t in already_running_tasks] shoalscrape_ids = [t.id for t in already_running_tasks] error_message = ('[ ! ] ShoalScrapeTask(s) {} attempted to start a' ' Celery worker with ID {} while they already had' ' Celery workers with IDs {}. Terminating worker with' ' Celery task ID {}'.format(shoalscrape_ids, current_task_id, other_celery_ids, current_task_id)) revoke(current_task_id, terminate=True) logger.info(error_message) raise Ignore() try: shoalscrape_task = ShoalScrapeTask.objects.get(id=shoalscrape_task_id) except ShoalScrapeTask.DoesNotExist: # Prevents Celery log spam for the defunct task. (different from VE) ptask_identifier = 'shoalscrape_task_{}'.format(shoalscrape_task_id) ptask = PeriodicTask.objects.get(name=ptask_identifier) task_error_log_template = ('ShoalScrapeTask #{} not found while ' 'starting. PeriodicTask #{}, {}, has been ' 'deleted.') logger.info( task_error_log_template.format(shoalscrape_task_id, ptask.id, ptask.name)) ptask.delete() raise Ignore() # This should come as early as possible after checking that the task is not # duplicated (ie, once current_task_id and shoalscrape_task are available): shoalscrape_task.current_task_id = current_task_id shoalscrape_task.save() ptask = shoalscrape_task.periodic_task # Prevents any other Celery workers from starting a task with the same ID: ptask.last_run_at = datetime.datetime.now() + datetime.timedelta(days=9999) ptask.save() shoalscrape_task.save() active_tasks = ShoalScrapeTask.objects.filter( Q(periodic_task__enabled=True) | Q(state=ShoalScrapeTask.IN_PROGRESS)).exclude(id=shoalscrape_task_id) # Checking active_tasks does not protect against multiple Celery workers # starting for the same ShoalScrapeTask; rather, it prevents multiple # ShoalScrapeTasks from starting. if active_tasks.exists(): ids = [each.id for each in active_tasks] error_message = ('[ ! ] Started while other ShoalScrape tasks were ' 'running: {}'.format(ids)) logger.info(error_message) shoalscrape_task.error = error_message shoalscrape_task.save() shoalscrape_task.set_state(ShoalScrapeTask.ERROR) shoalscrape_task.terminate_task(log_info=error_message) raise Ignore() server_tz = dj_tz.get_default_timezone() server_now = server_tz.localize(datetime.datetime.now()) shoalscrape_task.last_started_at = server_now shoalscrape_task.save() task_log_template = '[ + ] Attempting to start ShoalScrapeTask #{} at {}' logger.info( task_log_template.format(shoalscrape_task.id, shoalscrape_task.last_started_at)) results = 0 try: company = shoalscrape_task.company domain = shoalscrape_task.domain company_linkedin_id = shoalscrape_task.company_linkedin_id log_file_path = shoalscrape_task.path creds = shoalscrape_task.shoalscrape_creds user_agent = None if creds.scraper_user_agent: user_agent = creds.scraper_user_agent.user_agent_data username = creds.username password = creds.password results = shoalscrape.webserver_main(company, company_linkedin_id, domain, log_file_path, user_agent, username=username, password=password, scan_profiles=False, brute_force_emails=False) shoalscrape_task.set_state(ShoalScrapeTask.COMPLETE) shoalscrape_task.current_task_id = '' shoalscrape_task.save() except Exception as e: error = ('[ ! ] ShoalScrapeTask' ' #{} had an error: {}'.format(shoalscrape_task.id, e)) shoalscrape_task.set_state(ShoalScrapeTask.ERROR) shoalscrape_task.terminate_task(log_info=error) shoalscrape_task.error = error shoalscrape_task.save() logger.info(error) raise Ignore() return results