def import_wetmills(task): #pragma: no cover from .models import import_csv_wetmills from django.db import transaction transaction.enter_transaction_management() transaction.managed() try: task.task_id = import_wetmills.request.id task.import_log = "Started import at %s\n" % datetime.now() task.save() transaction.commit() wetmills = import_csv_wetmills(task.country, task.csv_file.file.name, task.created_by) task.log("Import finished at %s\n" % datetime.now()) task.log("%d wetmill(s) added." % len(wetmills)) transaction.commit() except Exception as e: import traceback traceback.print_exc() task.log("Error: %s" % e) transaction.commit() raise e finally: transaction.leave_transaction_management() return task
def finalize_season(task): #pragma: no cover from .models import SeasonAggregate from django.db import transaction transaction.enter_transaction_management() transaction.managed() try: task.task_id = finalize_season.request.id task.task_log = "Started finalizing season at %s\n" % datetime.now() task.save() transaction.commit() report_count = SeasonAggregate.calculate_for_season(task.season) task.log("Season finished at %s\n" % datetime.now()) task.log("%d finalized reports(s) included in season." % report_count) transaction.commit() except Exception as e: import traceback traceback.print_exc() task.log("Error: %s" % e) transaction.commit() raise e finally: transaction.leave_transaction_management() return task
def call_connector_task(task_id): """ call connector with args. """ connector = Connector() logger.info("call connector with task_id: %s", task_id) task = Task.objects.get(id=task_id) if task.status == States.DONE: return if task.status == States.PROCESSING: Picture.objects.filter(task_id=task.id).delete() else: task.status = States.PROCESSING task.save() try: data = connector.process_task(task) except Exception as e: return call_connector_task.retry(args=(task_id), countdown=60, exec=e, max_retries=2) for pic in data: p = Picture(link=pic["link"], pic_id=pic["id"], created_date=pic["date"], task=task) p.save() task.status = States.DONE task.save()
def calculateTask(task, sid, model): try: generate_mol_image(task) suite = SuiteTask.objects.get(sid=sid) map_model_name = get_model_name(model['model']) smile = task.file_obj.smiles.encode( 'utf-8') if task.file_obj.file_type != 'mol' else '' # smile, mol_fpath 输入只选择一种方式(优先smile) mol_fpath = os.path.join( settings.SETTINGS_ROOT, task.file_obj.file_obj.path) if not smile else None try: temperature = float(model.get('temperature')) except: temperature = DEFAULT_TEMPERATURE_ARGS chemistry_logger.info( 'PredictionModel calculating: model name(%s),' 'smile(%s) mol path(%s) temperature(%s)', map_model_name, smile, mol_fpath, temperature) # 后台模型计算入口 predict_results = prediction_model_calculate(map_model_name, smile, mol_fpath, temperature) if task.file_obj.file_type == 'mol': name = os.path.basename(mol_fpath).split('.')[0] else: name = smile result = predict_results[name][map_model_name] chemistry_logger.info('[task]result %s' % result) except KeyError: chemistry_logger.exception('still cannot support this model') result = None task.result_state = "We don't support this model now" task.status = StatusCategory.objects.get(category=STATUS_FAILED) suite.status_id = StatusCategory.objects.get(category=STATUS_FAILED) except Exception as e: chemistry_logger.exception('failed to submit task to prediction model') result = None task.result_state = str(e) task.status = StatusCategory.objects.get(category=STATUS_FAILED) suite.status_id = StatusCategory.objects.get(category=STATUS_FAILED) else: chemistry_logger.info("calculate Successfully in celery queue!") task.result_state = "Calculate Successfully!" task.status = StatusCategory.objects.get(category=STATUS_SUCCESS) suite.status_id = StatusCategory.objects.get(category=STATUS_WORKING) task.end_time = utils.get_real_now() task.results = json.dumps(result) suite.save() task.save() add_counter(suite.sid) return result
def calculateTask(task, sid, model): try: generate_mol_image(task) suite = SuiteTask.objects.get(sid=sid) map_model_name = get_model_name(model['model']) smile = task.file_obj.smiles.encode('utf-8') if task.file_obj.file_type != 'mol' else '' # smile, mol_fpath 输入只选择一种方式(优先smile) mol_fpath = os.path.join(settings.SETTINGS_ROOT, task.file_obj.file_obj.path) if not smile else None try: temperature = float(model.get('temperature')) except: temperature = DEFAULT_TEMPERATURE_ARGS chemistry_logger.info('PredictionModel calculating: model name(%s),' 'smile(%s) mol path(%s) temperature(%s)', map_model_name, smile, mol_fpath, temperature) # 后台模型计算入口 predict_results = prediction_model_calculate(map_model_name, smile, mol_fpath, temperature) if task.file_obj.file_type == 'mol': name = os.path.basename(mol_fpath).split('.')[0] else: name = smile result = predict_results[name][map_model_name] chemistry_logger.info('[task]result %s' % result) except KeyError: chemistry_logger.exception('still cannot support this model') result = None task.result_state = "We don't support this model now" task.status = StatusCategory.objects.get(category=STATUS_FAILED) suite.status_id = StatusCategory.objects.get(category=STATUS_FAILED) except Exception as e: chemistry_logger.exception('failed to submit task to prediction model') result = None task.result_state = str(e) task.status = StatusCategory.objects.get(category=STATUS_FAILED) suite.status_id = StatusCategory.objects.get(category=STATUS_FAILED) else: chemistry_logger.info("calculate Successfully in celery queue!") task.result_state = "Calculate Successfully!" task.status = StatusCategory.objects.get(category=STATUS_SUCCESS) suite.status_id = StatusCategory.objects.get(category=STATUS_WORKING) task.end_time = utils.get_real_now() task.results = json.dumps(result) suite.save() task.save() add_counter(suite.sid) return result
def import_hashfile_task(hashfile_id): hashfile = Hashfile.objects.get(id=hashfile_id) task = Task( time = datetime.datetime.now(), message = "Importing hash file %s..." % hashfile.name ) task.save() try: if hashfile.hash_type != -1: # if != plaintext task.message = "Importing hash file %s..." % hashfile.name task.save() Hashcat.insert_hashes(hashfile) task.message = "Comparing hash file %s to potfile..." % hashfile.name task.save() Hashcat.compare_potfile(hashfile) else: task.message = "Importing plaintext file %s..." % hashfile.name task.save() Hashcat.insert_plaintext(hashfile) except Exception as e: traceback.print_exc() finally: task.delete()
def remove_hashfile_task(hashfile_id): hashfile = Hashfile.objects.get(id=hashfile_id) task = Task(time=datetime.datetime.now(), message="Removing hash file %s..." % hashfile.name) task.save() try: Hashcat.remove_hashfile(hashfile) except Exception as e: traceback.print_exc() finally: task.delete()
def save_record(f, model, sid, source_type, smile=None, local_search_id=None): from chemistry.tasks import calculateTask, DEFAULT_TEMPERATURE_ARGS task = SingleTask() task.start_time = utils.get_real_now() task.sid = SuiteTask.objects.get(sid=sid) task.pid = str(uuid.uuid4()) task.model = ModelCategory.objects.get(category=model['model']) temperature = model.get('temperature') task.temperature = float(temperature) if temperature else DEFAULT_TEMPERATURE_ARGS if source_type == ORIGIN_UPLOAD: # here, f is ProcessedFile record instance f.file_source = FileSourceCategory.objects.get(category=source_type) f.file_type = "mol" task.file_obj = f f.save() elif source_type in (ORIGIN_SMILE, ORIGIN_DRAW): # here, f is a file path processed_f = ProcessedFile() obj = File(open(f, "r")) processed_f.title = os.path.basename(obj.name) processed_f.file_type = source_type processed_f.file_source = FileSourceCategory.objects.get(category=source_type) processed_f.file_obj = obj if smile: processed_f.smiles = smile if source_type == ORIGIN_SMILE and local_search_id is not None: processed_f.local_search_id = int(local_search_id) processed_f.save() task.file_obj = processed_f obj.close() else: raise ErrorCalculateType('Cannot recongize this source type') task.status = StatusCategory.objects.get(category=STATUS_WORKING) task.save() chemistry_logger.info('~~~~~~~~ t:%s' % task.start_time) calculateTask.delay(task, sid, model)
def import_scorecards(task): #pragma: no cover from .models import import_season_scorecards from django.db import transaction transaction.enter_transaction_management() transaction.managed() log = StringIO.StringIO() try: task.task_id = import_scorecards.request.id task.import_log = "Started import at %s\n\n" % datetime.now() task.save() transaction.commit() reports = import_season_scorecards(task.season, task.csv_file.file.name, task.created_by, log) task.log(log.getvalue()) task.log("Import finished at %s\n" % datetime.now()) task.log("%d scorecard(s) added." % len(reports)) transaction.commit() except Exception as e: import traceback traceback.print_exc(e) task.log("\nError: %s\n" % e) task.log(log.getvalue()) transaction.commit() raise e finally: transaction.leave_transaction_management() return task
def call_connector_task(task_id): """ call connector with args. """ connector = Connector() logger.info("call connector with task_id: %s",task_id) task = Task.objects.get(id=task_id) if task.status == States.DONE: return if task.status == States.PROCESSING: Picture.objects.filter(task_id=task.id).delete() else: task.status = States.PROCESSING task.save() try: data = connector.process_task(task) except Exception as e: return call_connector_task.retry(args=(task_id),countdown=60,exec=e,max_retries=2) for pic in data: p = Picture(link=pic["link"],pic_id=pic["id"],created_date=pic["date"],task=task) p.save() task.status = States.DONE task.save()
def run_recurring_tasks(): print "Looking for tasks..." now = datetime.datetime.now() tasks = RecurringTask.objects.filter( is_active=True, is_running=False).select_related("parent_job") active_tasks = [] for task in tasks: if task.period_unit == "minutes": if task.last_started_on is None or task.last_started_on < now - relativedelta(minutes=task.period): active_tasks.append(task) task.is_running = True task.save() elif task.period_unit == "hours": if task.last_started_on is None or task.last_started_on < now - relativedelta(hours=task.period): active_tasks.append(task) task.is_running = True task.save() elif task.period_unit == "days": if task.last_started_on is None or task.last_started_on < now - relativedelta(days=task.period): active_tasks.append(task) task.is_running = True task.save() for r in active_tasks: job = r.parent_job r.last_started_on = datetime.datetime.now() r.save() task = JobTask(parent_recurring_task=r) task.save() run_batch_job.delay(task)
def run_search_task(search_id): search = Search.objects.get(id=search_id) task = Task( time = datetime.datetime.now(), message = "Running search %s..." % search.name ) task.save() if os.path.exists(search.output_file): os.remove(search.output_file) try: search.status = "Running" search.output_lines = None search.processing_time = None search.save() search_info = json.loads(search.json_search_info) start_time = time.time() cursor = connection.cursor() args = [] columns = ["hashfile_id", "username", "password", "hash_type", "hash"] query = "SELECT %s FROM Hashcat_hash" % ",".join(columns) if "pattern" in search_info or not "all_hashfiles" in search_info or "ignore_uncracked" in search_info: query += " WHERE " if "pattern" in search_info: query_pattern_list = [] for pattern in search_info["pattern"].split(';'): query_pattern_list.append("username LIKE %s") args.append("%" + pattern + "%") query += "(" + " OR ".join(query_pattern_list) + ")" if not "all_hashfiles" in search_info or "ignore_uncracked" in search_info: query += " AND " if not "all_hashfiles" in search_info: query += "hashfile_id IN (%s)" % ','.join(['%s'] * len(search_info["hashfiles"])) args += [int(i) for i in search_info["hashfiles"]] if "ignore_uncracked" in search_info: query += " AND " if "ignore_uncracked" in search_info: query += "password IS NOT NULL" tmpfile_name = ''.join([random.choice(string.ascii_lowercase) for i in range(16)]) tmp_file = os.path.join(os.path.dirname(__file__), "..", "Files", "tmp", tmpfile_name) query += " INTO OUTFILE %s FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' LINES TERMINATED BY '\\n'" args.append(tmp_file) rows = cursor.execute(query, args) cursor.close() if os.path.exists(tmp_file): hash_types_dict = Hashcat.get_hash_types() hashfile_dict = {} for hashfile in Hashfile.objects.all(): hashfile_dict[hashfile.id] = hashfile.name with open(search.output_file, 'w', newline='') as out_csvfile: spamwriter = csv.writer(out_csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) with open(tmp_file, 'r', newline='') as csvfile: spamreader = csv.reader(csvfile, delimiter=',', quotechar='"') for row in spamreader: try: row[0] = hashfile_dict[int(row[0])] except KeyError: pass try: row[3] = hash_types_dict[int(row[3])]['name'] if int(row[3]) != -1 else "Plaintext" except KeyError: pass except ValueError: pass except IndexError: pass spamwriter.writerow(row) os.remove(tmp_file) end_time = time.time() search.status = "Done" search.output_lines = int(rows) search.processing_time = int(end_time - start_time) search.save() except Exception as e: traceback.print_exc() finally: task.delete()
task.num_rows, task.success_count, task.error_count = form.run(task, rows) except Exception, e: message = traceback.format_exc() subject = "[ActionKit Data Manager] Task %s (%s) failed :-(" % (task.id, name) else: message = "Num rows: %s. Success count: %s. Error count: %s." % ( task.num_rows, task.success_count, task.error_count) if task.error_count: subject = "[ActionKit Data Manager] Task %s (%s) completed with errors =/" % (task.id, name) else: subject = "[ActionKit Data Manager] Task %s (%s) succeeded =)" % (task.id, name) message += "\n\nCheck it out here: http://%s/admin/main/jobtask/%s/" % (settings.SITE_DOMAIN, task.id) message += "\nThe job configuration is here: http://%s/admin/main/batchjob/%s/" % (settings.SITE_DOMAIN, job.id) task.completed_on = datetime.datetime.now() task.save() if recurrence is not None: recurrence.is_running = False recurrence.save() num = send_mail(subject, message, settings.DEFAULT_FROM_EMAIL, [job.created_by.email] + [i[1] for i in settings.ADMINS], fail_silently=False) print "Sent %s mails with subject %s; job %s completed; %s rows" % (num, subject, job.id, task.num_rows) @periodic_task(run_every=datetime.timedelta(seconds=60)) def run_recurring_tasks(): print "Looking for tasks..."