class Method(models.Model):
path = models.ForeignKey(Path, on_delete=models.CASCADE)
name = models.CharField(max_length=4, choices=HTTP_METHODS)
summary = models.CharField(max_length=200, blank=True)
description = models.CharField(max_length=200, blank=True)
parameters = fields.PickledObjectField(null=True)
input_types = fields.PickledObjectField(null=True)
output_types = fields.PickledObjectField(null=True)
def get_path_name(self):
path = Path.objects.get(id=self.path_id)
return path.name
class Game(models.Model):
num = models.IntegerField(primary_key=True)
name = models.CharField(max_length=150)
played = models.BooleanField(default=False)
jeopardy_questions = fields.PickledObjectField()
jeopardy_answers = fields.PickledObjectField()
double_jeopardy_questions = fields.PickledObjectField()
double_jeopardy_answers = fields.PickledObjectField()
final_category = models.CharField(max_length=250)
final_clue = models.CharField(max_length=500)
final_answer = models.CharField(max_length=500)
state = State()
class PluginInstance(models.Model):
"""Instance of a plugin (including settings)."""
name = models.CharField(max_length=64)
description = models.TextField(max_length=254, blank=True, null=True)
plugin = models.ForeignKey(Plugin)
statuses = models.ManyToManyField(Status, blank=True)
config = picklefield.PickledObjectField(null=True)
created_on = models.DateTimeField(auto_now_add=True)
modified_on = models.DateTimeField(auto_now=True)
@property
def component(self):
"""Return the component associated with this instance."""
components = self.component_set.all()
if not components:
return None
return components[0]
def __str__(self):
return '%s <Component: %s, Updated: %s>' % (
self.plugin.name,
self.component,
self.modified_on
)
class Site(models.Model):
name = models.CharField(max_length=128)
user = models.ForeignKey(User, on_delete=models.CASCADE)
props = models.ManyToManyField(Property)
data = fields.PickledObjectField()
def __str__(self):
return self.name
class Operation(models.Model):
name = models.CharField(max_length=200)
endpoint = models.ForeignKey(Endpoint, on_delete=models.CASCADE, null=True)
documentation = models.TextField(editable=False)
parameters = fields.PickledObjectField()
soap_action = models.CharField(max_length=1000, default='')
def __str__(self):
return json.dumps(self.parameters)
class Path(models.Model):
web_service = models.ForeignKey(RestWebService, on_delete=models.CASCADE)
name = models.CharField(max_length=100)
methods = fields.PickledObjectField(null=True, choices=HTTP_METHODS)
def get_base_url(self):
web_service = RestWebService.objects.get(id=self.web_service_id)
return web_service.base_url
def get_methods_accept_xml(self):
methods_accept_xml = []
for method in self.method_set.all():
if method.input_types:
if 'application/xml' in method.input_types:
methods_accept_xml.append(method)
print(method.name)
return methods_accept_xml
class CremeModel(BaseModel):
name = models.TextField(unique=True)
pipeline = picklefield.PickledObjectField()
class Meta:
db_table = 't_creme_models'
verbose_name_plural = 'models'
def fit_one(self, x, y):
"""Attempts to update the model and indicates if the update was successful or not."""
return CremeModel.objects\
.filter(id=self.id, updated_at=self.updated_at)\
.update(pipeline=self.pipeline.fit_one(x, y)) > 0
def predict_one(self, x):
return (self.pipeline.predict_one(x),
CremeModel.objects.filter(id=self.id,
updated_at=self.updated_at).update(
pipeline=self.pipeline) > 0)
def __str__(self):
return f'Model {self.name}'
class OcrTask(models.Model):
"""
OCR Task object.
"""
STATUS_CHOICES = (
("INIT", "Initialising"),
("PENDING", "Pending"),
("STARTED", "Started"),
("RETRY", "Retry"),
("SUCCESS", "Success"),
("FAILURE", "Failure"),
)
user = models.ForeignKey(User)
batch = models.ForeignKey(Batch,
related_name="tasks",
blank=True,
null=True)
project = models.ForeignKey(Project,
related_name="tasks",
blank=True,
null=True)
task_id = models.CharField(max_length=100)
task_name = models.CharField(max_length=100)
page_name = models.CharField(max_length=255)
status = models.CharField(max_length=20, choices=STATUS_CHOICES)
lines = models.IntegerField(blank=True, null=True)
progress = models.FloatField(default=0.0, blank=True, null=True)
args = fields.PickledObjectField(blank=True, null=True)
kwargs = fields.PickledObjectField(blank=True, null=True)
error = fields.PickledObjectField(blank=True, null=True)
traceback = models.TextField(blank=True, null=True)
created_on = models.DateTimeField(editable=False)
updated_on = models.DateTimeField(blank=True, null=True, editable=False)
def save(self):
if not self.id:
self.created_on = datetime.datetime.now()
else:
self.updated_on = datetime.datetime.now()
super(OcrTask, self).save()
def abort(self):
"""
Abort a task.
"""
if not self.is_active():
return
asyncres = AbortableAsyncResult(self.task_id)
if self.is_abortable():
asyncres.abort()
if asyncres.is_aborted():
self.status = "ABORTED"
self.save()
celery.task.control.revoke(self.task_id,
terminate=True,
signal="SIGTERM")
def run(self, task_name=None, asyncronous=True, untracked=False, **kwargs):
"""
Run the task in a blocking manner and return
the sync object.
"""
tname = task_name if task_name is not None else self.task_name
if untracked:
tname = "_%s" % tname
celerytask = celery.registry.tasks[tname]
func = celerytask.apply_async if asyncronous else celerytask.apply
kwds = self.kwargs
kwds.update(kwargs)
return func(args=self.args, **kwds)
def retry(self):
"""
Retry the Celery job.
"""
if self.is_abortable():
self.abort()
self.task_id = self.get_new_task_id()
self.status = "RETRY"
self.progress = 0
self.save()
self.kwargs["task_id"] = self.task_id
celerytask = celery.registry.tasks[self.task_name]
celerytask.apply_async(args=self.args, **self.kwargs)
def is_batch_task(self):
"""
Whether task is part of a batch.
"""
return self.batch is None
def is_revokable(self):
"""
Whether or not a given status allows
revoking (cancelling) a task.
"""
return self.status in ("INIT", "PENDING")
def is_abortable(self):
"""
Whether we can cancel execution.
"""
return self.status in ("STARTED", "RETRY")
def is_active(self):
"""
The task is running or awaiting running.
"""
return self.status in ("INIT", "PENDING", "RETRY", "STARTED")
@classmethod
def run_celery_task(cls, taskname, args, taskkwargs={}, **kwargs):
"""
Run an arbitary Celery task.
"""
if kwargs.get("untracked", False):
taskname = "_%s" % taskname
task = celery.registry.tasks[taskname]
func = task.apply_async if kwargs.get("asyncronous", False) \
else task.apply
return func(args=args, kwargs=taskkwargs, **kwargs)
@classmethod
def run_celery_task_multiple(cls, taskname, tasks, **kwargs):
"""
Optimised method for running multiple Celery tasks
(uses the same celery publisher.)
"""
if len(tasks) == 0:
return []
celerytask = celery.registry.tasks[taskname]
publisher = celerytask.get_publisher(connect_timeout=5)
func = celerytask.apply_async if kwargs.get("asyncronous", True) \
else celerytask.apply
results = []
try:
for task in tasks:
results.append(
func(args=task.args,
kwargs=task.kwargs,
publisher=publisher,
task_id=task.task_id,
**kwargs))
finally:
publisher.close()
publisher.connection.close()
return results
@classmethod
def revoke_celery_task(cls, task_id, kill=True):
"""
Kill a Celery task.
"""
celery.task.control.revoke(task_id, terminate=kill, signal="SIGTERM")
@classmethod
def get_celery_result(cls, task_id):
"""
Proxy for fetching Celery results.
"""
return celery.result.AsyncResult(task_id)
@classmethod
def get_new_task_id(cls):
"""
Get a unique id for a new page task, given it's
file path.
"""
return str(uuid.uuid1())
def get_absolute_url(self):
"""URL to view an object detail"""
return "/ocrtasks/detail/%d/" % self.pk
def get_delete_url(self):
"""url to update an object detail"""
return "/ocrtasks/delete/%d/" % self.pk
@classmethod
def get_list_url(cls):
"""URL to view the object list"""
return "/ocrtasks/list/"
class Request(models.Model):
url = models.URLField(null=True)
http_method = models.CharField(max_length=4, choices=HTTP_METHODS)
data = fields.PickledObjectField(null=True, blank=True)
files = fields.PickledObjectField(null=True)
headers = fields.PickledObjectField(null=True)
category = models.CharField(max_length=13, choices=REQUESTS_TYPES)
# If category is not malicious there this field will be null
attack_type = models.ForeignKey(Attack,
on_delete=models.CASCADE,
null=True)
# If category is malicious and attack type is injection this field will contain the used pattern
pattern = models.CharField(max_length=200, null=True)
def __str__(self):
return self.data
def get_proxies(self):
url = 'https://free-proxy-list.net/'
response = requests.get(url)
parser = fromstring(response.text)
proxies = []
for i in parser.xpath('//tbody/tr')[:10]:
if i.xpath('.//td[7][contains(text(),"yes")]'):
# Grabbing IP and corresponding PORT
proxy = ":".join([
i.xpath('.//td[1]/text()')[0],
i.xpath('.//td[2]/text()')[0]
])
proxies.append(proxy)
return proxies
def send_request(self, files=None):
timeout = TIMEOUT
try:
# s = requests.Session()
# s.proxies = {"http": "http://61.233.25.166:80"}
if self.http_method == 'POST':
if files is not None:
resp = requests.post(self.url,
files=files,
headers=self.headers,
timeout=timeout)
else:
resp = requests.post(self.url,
data=self.data,
headers=self.headers,
timeout=timeout)
elif self.http_method == 'GET':
resp = requests.get(self.url,
params=self.data,
headers=self.headers,
timeout=timeout)
elif self.http_method == 'PUT':
resp = requests.put(self.url,
data=self.data,
headers=self.headers,
timeout=timeout)
elif self.http_method == 'DELETE':
resp = requests.delete(self.url,
data=self.data,
headers=self.headers,
timeout=timeout)
elif self.http_method == 'PATCH':
resp = requests.patch(self.url,
data=self.data,
headers=self.headers,
timeout=timeout)
elif self.http_method == 'HEAD':
resp = requests.head(self.url,
data=self.data,
headers=self.headers,
timeout=timeout)
elif self.http_method == 'OPTIONS':
resp = requests.options(self.url,
data=self.data,
headers=self.headers,
timeout=timeout)
response = Response()
response.content = html.unescape(resp.text)
response.http_status_code = resp.status_code
response.time_to_first_byte = resp.elapsed.total_seconds()
except requests.exceptions.ConnectionError:
print('Connection Error !')
response = Response()
response.content = ''
response.http_status_code = -1
response.time_to_first_byte = -1
except requests.exceptions.RequestException:
print("Requests Exception has occured !")
response = Response()
response.content = ''
response.http_status_code = -1
response.time_to_first_byte = -1
except urllib3.exceptions.HTTPError:
print("Urllib3 New Connection Error !")
response = Response()
response.content = ''
response.http_status_code = -1
response.time_to_first_byte = -1
return response
class InjAttack(Attack):
def __init__(self, *args, **kwargs):
super(InjAttack, self).__init__(*args, **kwargs)
self.family = 'Inj'
patterns = fields.PickledObjectField(null=True)
class UserFilter(Filter):
title = models.TextField(max_length=128, default="User-defined filter")
query = fields.PickledObjectField()
owner = models.ForeignKey(User, on_delete=models.CASCADE)
class RestWebService(WebService):
base_url = models.URLField(verbose_name='Base URL')
paths = fields.PickledObjectField(null=True)
class Meta:
ordering = ('id',)
class SwaggerResponse(models.Model):
method = models.ForeignKey(Method, on_delete=models.CASCADE)
http_code = models.CharField(max_length=3, choices=HTTP_STATUS_CODES)
description = models.TextField(editable=False)
schema = fields.PickledObjectField(null=True)
