class Video(MongoModel):
'''Model to maintain videos.
Attributes:
createdOn: Datetime object
updatedOn: Datetime object
title
description
publishedAt
thumbnailURL
channelTitle
videoURL
'''
createdOn = fields.DateTimeField(required=False, default=lambda: datetime.now())
updatedOn = fields.DateTimeField(required=False, default=lambda: datetime.now())
title = fields.CharField(required=True)
description = fields.CharField(blank=True)
publishedAt = fields.DateTimeField(required=False, default=lambda: datetime.now())
thumbnailURL = fields.URLField(required=False)
channelTitle = fields.CharField(required=True)
videoURL = fields.URLField(required=False)
objects = VideoManager()
class Meta:
collection_name = 'Videos'
indexes = [
IndexModel(
[
('publishedAt', DESCENDING),
],
background=True
)
]
final = True
class FileModel(MongoModel):
fileId = fields.CharField(primary_key=True, required=True)
processingStatus = fields.CharField(required=True)
fileName = fields.CharField(required=True)
fileLength = fields.IntegerField(required=False)
mp3Path = fields.URLField(required=False)
originalFilePath = fields.URLField(required=False)
seriesTitle = fields.CharField(required=False)
segments = fields.ListField(field=fields.DictField(), required=False)
class Product(MongoModel):
image_url = fields.URLField()
product_url = fields.URLField(
primary_key=True) # this is the unique ID for a product.
price = fields.FloatField()
title = fields.CharField()
store = fields.CharField()
class Meta:
connection_alias = "main"
class NotedMedia(MongoModel):
# because primary_key is True, we will need to query this
# field using the label _id
media_id = fields.CharField(primary_key=True)
media_src = fields.URLField()
media_title = fields.CharField()
media_img = fields.URLField()
# email = fields.EmailField()
ts_start = fields.ListField(field=fields.FloatField())
ts_end = fields.ListField(field=fields.FloatField())
body = fields.ListField(field=fields.CharField())
ts_insert = fields.ListField(field=fields.DateTimeField())
class Request(MongoModel):
email = fields.EmailField()
src_url = fields.CharField() # handles base64 encodings
link_url = fields.URLField()
page_url = fields.URLField()
email_src_concat = fields.CharField(
primary_key=True) # unique ID is combined email + img src
time_received = fields.DateTimeField()
time_due = fields.DateTimeField()
priority_list = fields.ListField(fields.CharField(), default=[])
class Meta:
connection_alias = "main"
class CTF(MongoModel):
name = fields.CharField(required=True)
description = fields.CharField()
created_at = fields.DateTimeField(required=True)
finished_at = fields.DateTimeField()
start_date = fields.DateTimeField()
end_date = fields.DateTimeField()
url = fields.URLField()
username = fields.CharField()
password = fields.CharField()
challenges = fields.EmbeddedDocumentListField(Challenge,
default=[],
blank=True)
pending_reminders = fields.ListField(blank=True, default=[])
def status(self, members_joined_count):
description_str = self.description + "\n" if self.description else ""
solved_count = len(
list(filter(lambda x: x.solved_at is not None, self.challenges)))
total_count = len(self.challenges)
status = (
f":triangular_flag_on_post: **{self.name}** ({members_joined_count} Members joined)\n{description_str}"
+ f"```CSS\n{draw_bar(solved_count, total_count, style=5)}\n" +
f" {solved_count} Solved / {total_count} Total")
if self.start_date:
fmt_str = "%d/%m %H:\u200b%M"
start_date_str = self.start_date.strftime(fmt_str)
end_date_str = self.end_date.strftime(
fmt_str) if self.end_date else "?"
status += f"\n {start_date_str} - {end_date_str}\n"
status += "```"
return status
def credentials(self):
response = f":busts_in_silhouette: **Username**: {self.username}\n:key: **Password**: {self.password}"
if self.url is not None:
response += f"\n\nLogin Here: {self.url}"
return response
def challenge_summary(self):
if not self.challenges:
return "No challenges found. Try adding one with `!ctf addchallenge <name> <category>`"
solved_response, unsolved_response = "", ""
for challenge in self.challenges:
challenge_details = f'**{escape_md(challenge.name[len(self.name)+1:])}** [{", ".join(challenge.tags)}]'
if challenge.solved_at:
solved_response += f':white_check_mark: {challenge_details} Solved by: [{", ".join(challenge.solved_by)}]\n'
else:
unsolved_response += f':thinking: {challenge_details} Attempted by: [{escape_md(", ".join(challenge.attempted_by))}]\n'
return (f"\\>>> Solved\n{solved_response}" +
f"\\>>> Unsolved\n{unsolved_response}")
class Meta:
collection_name = "ctf"
ignore_unknown_fields = True
class RecipeSource(EmbeddedMongoModel):
name = fields.CharField()
url = fields.URLField()
image = fields.ImageField()
class Link(EmbeddedMongoModel):
label = fields.CharField(required=True,
verbose_name='link_label',
mongo_name='label')
url = fields.URLField(required=True,
verbose_name='link_url',
mongo_name='url')
class Trails(MongoModel):
"""
A model defining trails entity undertaken by hikers. It has embedded
comments which reference the hiker.
"""
trail_name = fields.CharField(max_length=100, required=True)
distance = fields.FloatField(max_value=50, required=True)
elevation = fields.FloatField(required=True)
route_type = fields.CharField(max_length=20,
choices=('Point to Point', 'Loop',
'Out and Back'),
required=True)
difficulty = fields.CharField(max_length=20,
choices=('Easy', 'Moderate', 'Difficult'),
required=True)
description = fields.CharField(max_length=600, required=True)
centrepoint = fields.PointField(required=False)
waypoints = fields.LineStringField(required=False)
image = fields.URLField(required=True)
embed_route = fields.CharField(max_length=800, required=True)
location = fields.ReferenceField(Location)
comments = fields.EmbeddedDocumentListField('Comment', required=False)
def __str__(self):
return self.trail_name
class Course(MongoModel):
course_id = fields.CharField()
course_name = fields.CharField()
textbook = fields.URLField()
topics = fields.ListField()
students = fields.EmbeddedDocumentListField(Student)
quiz = fields.EmbeddedDocumentListField(Quiz)
class Comp(MongoModel):
name = fields.CharField(required=True)
description = fields.CharField()
created_at = fields.DateTimeField(required=True)
deadline = fields.DateTimeField()
merger_deadline = fields.DateTimeField()
url = fields.URLField()
team_name = fields.CharField(required=False)
team_members = fields.ListField(fields.CharField())
max_team_size = fields.IntegerField()
max_daily_subs = fields.IntegerField()
subs_today = fields.IntegerField(default=0)
finished_on = fields.DateTimeField()
def status(self):
members = ",".join(self.team_members)
deadline = self.deadline if self.deadline > datetime.datetime.now(
) else "Passed"
merger_deadline = self.merger_deadline if self.merger_deadline > datetime.datetime.now(
) else "Passed"
status = (
f":triangular_flag_on_post:**\t{self.name}**\n***{self.description}.***\n**Merger Deadline:**\t{merger_deadline}"
+
f"\n**Deadline:**\t{deadline}\n**Team Name:**\t{self.team_name}\n:busts_in_silhouette:\t{members}\n"
+ f"**Team Size:**\t{self.max_team_size}\n:link:\t{self.url}")
return status
class GoogleFieldsModel(EmbeddedMongoModel):
name = fields.CharField()
given_name = fields.CharField()
family_name = fields.CharField()
picture = fields.URLField()
email = fields.EmailField()
email_verified = fields.BooleanField()
token = fields.CharField()
class Website(EnsureEntry, MongoModel):
"""Website Model"""
ensure_fields = ['website_uid']
# -------------------------------------------------------------------------
# Model Field Definitions
# -------------------------------------------------------------------------
website_uid = fields.CharField(required=True)
website_uid._ensure_lookup_field = True # Make field a lookup field
website = fields.CharField(required=True)
uri = fields.URLField()
# -------------------------------------------------------------------------
# Document Version to keep track of model migrations
# -------------------------------------------------------------------------
doc_version = fields.FloatField(required=True, default=1.0)
def clean_fields(self, *args, **kwargs):
"""Adjust Fields before cleaning"""
# -------------------------------------------------------------------------
# Create Unique Website ID if not set
# -------------------------------------------------------------------------
if not self.website_uid and self.website:
self.website_uid = self.website
# -------------------------------------------------------------------------
# Clean Website ID
# -------------------------------------------------------------------------
if self.website_uid:
self.website_uid = utils.cleanStringForUID(self.website_uid)
# -------------------------------------------------------------------------
# Continue original function
# -------------------------------------------------------------------------
super(Website, self).clean_fields(*args, **kwargs)
class Meta:
"""Meta class for Website Model"""
collection_name = "websites"
# -------------------------------------------------------------------------
# Helper Functions
# -------------------------------------------------------------------------
@classmethod
def get_lookup_arguments(cls, website_id):
"""Convert website_id into valid ObjectId if exists"""
lookup_args = {}
if check_is_valid_object_id(website_id):
lookup_args['_id'] = ObjectId(website_id)
elif isinstance(website_id, basestring):
found = cls.objects.get(
{'website_uid': website_id.lower().strip()})
lookup_args['_id'] = found.pk
else:
raise ValueError("'%s' is not valid website lookup value" %
(website_id))
return lookup_args
class Agents(MongoModel):
email = fields.EmailField(required=True)
password = fields.CharField(required=True)
name = fields.CharField(required=True)
primary_image = fields.URLField(required=True)
background_image = fields.URLField(required=True)
location = fields.PointField(required=True)
category = fields.CharField(required=True)
phone = fields.CharField(required=True)
feedback = fields.FloatField(blank=True)
totalreviews = fields.IntegerField(blank=True)
redeemptstoggle = fields.BooleanField(required=True)
starttime = fields.DateTimeField(required=True)
endtime = fields.DateTimeField(required=True)
#image url , offer template url , location , category , phone ,
#reward points , offers , punch cards , feedback , birthtday
created_at = fields.DateTimeField()
updated_at = fields.DateTimeField()
class Meta:
indexes = [
pymongo.IndexModel([('email', pymongo.ALL)],
name="AgentEmailUniqueIndex",
unique=True),
pymongo.IndexModel([("location", pymongo.GEOSPHERE)]),
pymongo.IndexModel([('name', pymongo.ALL),
('consumer_id', pymongo.ALL)],
name="NameIndex")
] # indexes
# asd= rest.objects.raw({'location': {'$near': SON([('$geometry', SON([('type', 'Point'), ('coordinates', [-122.406417,37.785834])])), ('$maxDistance', 500)])}})
# asd = rest.objects.raw({"location":{"$near": {"$geometry": {"type":"Point","coordinates":[-122.406417,37.785834]},"$maxDistance":500}}})
# for i in asd:
# print i._data
class CTF(MongoModel):
name = fields.CharField(required=True)
description = fields.CharField()
created_at = fields.DateTimeField(required=True)
finished_at = fields.DateTimeField()
url = fields.URLField()
username = fields.CharField()
password = fields.CharField()
challenges = fields.EmbeddedDocumentListField(Challenge, default=[])
def status(self, members_joined_count):
fmt_str = '%d/%m/%Y-%H:%M:%S'
start_date_str = self.created_at.strftime(fmt_str)
end_date_str = self.finished_at.strftime(
fmt_str) if self.finished_at else 'Live'
description_str = self.description if self.description else '_No description set_'
return f':triangular_flag_on_post: **{self.name}** ({members_joined_count} Members joined)\n{description_str}\n' +\
f'{len(list(filter(lambda x: x.solved_at != None, self.challenges)))} Solved / {len(self.challenges)} Total\n' +\
f'[{start_date_str} - {end_date_str}]\n'
def credentials(self):
return f'Username: {self.username}\nPassword: {self.password}'
def challenge_summary(self):
if not self.challenges:
return [
'No challenges found. Try adding one with `!ctf addchallenge <name> <category>`'
]
solved_response, unsolved_response = '', ''
for challenge in self.challenges:
challenge_details = f'**{challenge.name[len(self.name)+1:]}** [{", ".join(challenge.tags)}]'
if challenge.solved_at:
solved_response += f':white_check_mark: {challenge_details} Solved by: [{", ".join(challenge.solved_by)}]\n'
else:
unsolved_response += f':thinking: {challenge_details} Attempted by: [{", ".join(challenge.attempted_by)}]\n'
div = "-" * len(self.name) * 2
summary = f'>>> Solved\n{solved_response}' + f'>>> Unsolved\n{unsolved_response}'
summary_list = []
while len(summary) > 1900: # Embed has a limit of 2048 chars
idx = summary.index('\n', 1900)
summary_list.append(summary[:idx])
summary = summary[idx:]
summary_list.append(summary)
return summary_list
class ShelfImage(MongoModel):
# Drawback: Ties primary key to cloudinary but migration is easy.
cloudinary_public_id = fields.CharField(primary_key=True)
cloudinary_url = fields.URLField()
review_status = fields.CharField(
choices=['reviewed', 'to-review'],
default='to-review')
class Meta:
collection_name = 'shelf_images'
def to_json(self):
return {
"id": self.cloudinary_public_id,
"url": self.cloudinary_url,
"reviewStatus": self.review_status
}
class Hiker(MongoModel):
"""
A model defining hiker, with hiker attribute, first name, last name,
country of origin and e-mail. Profile pic is a url field that stores
the link to the hiker's profile photo. Trails completed field refers
to the no. of trails completed by hiker currently.
"""
fname = fields.CharField(min_length=2, max_length=30)
lname = fields.CharField(min_length=2, max_length=20)
username = fields.CharField(min_length=5, max_length=20)
origin = fields.CharField(max_length=30)
email = fields.EmailField()
trails_completed = fields.IntegerField(min_value=0, default=0)
profile_pic = fields.URLField(required=True)
def __str__(self):
return f'{self.fname}({self.lname})'
class ProductImage(MongoModel):
id = fields.CharField(primary_key=True)
shelf_image_id = fields.CharField(required=True)
shelf_image_url = fields.URLField(required=True)
upc = fields.CharField(default='000000000000')
crop = fields.EmbeddedDocumentField(ImageCrop)
bbox = fields.EmbeddedDocumentField(ImageBbox)
review_status = fields.CharField(choices=('reviewed', 'to-review'))
class Meta:
collection_name = 'product_images'
indexes = [
IndexModel([
('upc', pymongo.ASCENDING)
]),
IndexModel([
('shelf_image_id', pymongo.ASCENDING)
])
]
def to_json(self):
return {
"id": self.id,
"shelfImage": {
"id": self.shelf_image_id,
"url": self.shelf_image_url
},
"crop": {
"x": self.crop.x if self.crop else None,
"y": self.crop.y if self.crop else None,
"w": self.crop.w if self.crop else None,
"h": self.crop.h if self.crop else None
},
"bbox": {
"x": self.bbox.x if self.bbox else None,
"y": self.bbox.y if self.bbox else None,
"w": self.bbox.w if self.bbox else None,
"h": self.bbox.h if self.bbox else None
},
"upc": self.upc,
"reviewStatus": self.review_status
}
class Article(MongoModel):
"""
Model example to work with Mongo DB
List of possible fields
https://pymodm.readthedocs.io/en/0.4.0/api/index.html#model-fields
"""
#_id = fields.ObjectIdField(primary_key=True)
truth = fields.BooleanField() # Mostly True / Mostly False
created = fields.DateTimeField(blank=True)
url = fields.URLField()
authors = fields.ListField(field=fields.CharField(), blank=True)
date = fields.DateTimeField(blank=True)
source = fields.CharField(blank=True)
tags = fields.ListField(field=fields.CharField(), blank=True)
location = fields.PointField(blank=True)
title = fields.CharField(blank=True)
text = fields.CharField(blank=True)
abstract = fields.CharField(blank=True)
keywords = fields.ListField(field=fields.CharField(), blank=True)
similar_to = fields.ListField(field=fields.ObjectIdField(), blank=True)
class Meta:
connection_alias = APP_DB
class Product(EnsureEntry, EnsureReferencedLookup, MongoModel):
"""Product Model"""
ensure_fields = ['sku', 'website', 'crawled_at']
# -------------------------------------------------------------------------
# Model Field Definitions
# -------------------------------------------------------------------------
sku = fields.CharField(required=True)
name = fields.CharField(required=True)
product_type = fields.CharField(default='')
url = fields.URLField(required=True)
path = fields.CharField()
crawled_at = fields.DateTimeField(required=True)
price = fields.FloatField()
on_sale = fields.BooleanField(required=True, default=False)
discount_percentage = fields.FloatField(default=0.0)
properties = fields.DictField(required=True, default={})
# -------------------------------------------------------------------------
# Reference Brand Model with Cascade if referenced model is deleted
# -------------------------------------------------------------------------
brand = fields.ReferenceField(Brand,
required=True,
verbose_name="Brand",
on_delete=fields.ReferenceField.CASCADE)
brand._ref_lookup = True # Enable reference lookup if set value is string
# -------------------------------------------------------------------------
# Reference Website Model with Cascade if referenced model is deleted
# -------------------------------------------------------------------------
website = fields.ReferenceField(Website,
required=True,
verbose_name="Website",
on_delete=fields.ReferenceField.CASCADE)
website._ref_lookup = True # Enable reference lookup if set value is string
listings = fields.EmbeddedDocumentListField(
ProductListingItem,
default=[],
required=False,
)
# -------------------------------------------------------------------------
# Document Version to keep track of model migrations
# -------------------------------------------------------------------------
doc_version = fields.FloatField(required=True, default=1.0)
def clean(self):
"""Clean Values"""
# -------------------------------------------------------------------------
# Determine if discount was set and that on_sale is correctly set
# -------------------------------------------------------------------------
if self.discount_percentage and isinstance(
self.discount_percentage,
float) and self.discount_percentage > 0.0:
self.on_sale = True
# -------------------------------------------------------------------------
# Make sure that discount_percentage is zero if not on_sale
# -------------------------------------------------------------------------
if not self.on_sale:
self.discount_percentage = 0.0
# -------------------------------------------------------------------------
# Extract path from url if path wasn't set
# -------------------------------------------------------------------------
if self.url and not self.path:
parsed_url = urlparse(self.url)
self.path = parsed_url.path
# # -------------------------------------------------------------------------
# # Clean Extra Properties
# # -------------------------------------------------------------------------
# clean_props = {}
# for pk, pv in self.properties.items():
# clean_props[utils.cleanKeyForDict(pk)] = pv
# self.properties = clean_props
class Meta:
"""Meta class for Product Model"""
collection_name = "products"
# # -------------------------------------------------------------------------
# # Create Unique Index
# # -------------------------------------------------------------------------
# indexes = [
# IndexModel([('sku', TEXT), ('brand', TEXT), ('website', TEXT)], name="uprod_idx", unique=True),
# ]
# -------------------------------------------------------------------------
# Helper Functions
# -------------------------------------------------------------------------
@classmethod
def get_lookup_arguments(cls, product_id, website_id=None):
"""Convert product_id into valid ObjectId if exists"""
lookup_args = {}
if check_is_valid_object_id(product_id):
lookup_args['_id'] = ObjectId(product_id)
elif isinstance(product_id,
basestring) and check_is_valid_object_id(website_id):
regexp_lookup = r'^(%s)$' % (re.escape(product_id))
found = cls.objects.get({
'sku':
re.compile(regexp_lookup, re.IGNORECASE),
'website':
ObjectId(website_id),
})
lookup_args['_id'] = found.pk
else:
raise ValueError("'%s' is not valid product lookup value" %
(product_id))
return lookup_args
class CTFModel(MongoModel):
name = fields.CharField(required=True)
guild_id = fields.IntegerField()
category_id = fields.IntegerField()
role_id = fields.IntegerField()
description = fields.CharField()
created_at = fields.DateTimeField(required=True)
finished_at = fields.DateTimeField()
start_date = fields.DateTimeField()
end_date = fields.DateTimeField()
url = fields.URLField()
username = fields.CharField()
password = fields.CharField()
challenges = fields.EmbeddedDocumentListField(Challenge,
default=[],
blank=True)
pending_reminders = fields.ListField(blank=True, default=[])
tags = fields.ListField(fields.CharField(), default=[], blank=True)
def status(self, members_joined_count):
description_str = self.description + "\n" if self.description else ""
solved_count = len(
list(filter(lambda x: x.solved_at is not None, self.challenges)))
total_count = len(self.challenges)
status = (
f":triangular_flag_on_post: **{self.name}** ({members_joined_count} Members joined)\n{description_str}"
# + f"```CSS\n{draw_bar(solved_count, total_count, style=5)}\n"
+ f" {solved_count} Solved / {total_count} Total")
if self.start_date:
fmt_str = "%d/%m %H:\u200b%M"
start_date_str = self.start_date.strftime(fmt_str)
end_date_str = self.end_date.strftime(
fmt_str) if self.end_date else "?"
status += f"\n {start_date_str} - {end_date_str}\n"
status += "```"
return status
def credentials(self):
response = f":busts_in_silhouette: **Username**: {self.username}\n:key: **Password**: {self.password}"
if self.url is not None:
response += f"\n\nLogin Here: {self.url}"
return response
def get_chal_hash(self, name):
return hashlib.sha1(
f'{self.category_id}{name}'.encode()).hexdigest()[:7]
def get_challenge(self, name):
challenge = next(
(c for c in self.challenges
if c.name == name or self.get_chal_hash(c.name) == name), None)
if not challenge:
raise NotFound("Challenge not found.")
return challenge
# return next((c for c in self.challenges if c.name == name or self.get_chal_hash(c.name) == name), None)
def challenge_summary(self, category):
if len(self.challenges) == 0:
raise NotFound(
"No challenges found. Add one with `>challenge add <name> <category>`"
)
solved_response, unsolved_response = "", ""
challenges = self.challenges
if category and category != 'all':
challenges = filter(lambda c: category in c.tags, challenges)
challenges = sorted(challenges,
key=lambda c:
(c.tags, c.name, c.solved_at or c.created_at))
for challenge in challenges:
chal_hash = self.get_chal_hash(challenge.name)
challenge_details = f'[{chal_hash} :: {challenge.name}]'
notes_url = f'[notes]({challenge.notebook_url})'
flag = f'{{{challenge.flag or ""}}}'
tags = ",".join(challenge.tags)
if challenge.solved_at:
solved_response += f'> {challenge_details}({tags}) <{",".join(challenge.solved_by)}> {flag}\n'
else:
if len(challenge.working_on) > 0:
unsolved_response += f'* {challenge_details}({tags}) <{",".join(challenge.working_on)}> {flag}\n'
else:
unsolved_response += f'< {challenge_details}({tags}) < -- > {flag}\n'
return solved_response, unsolved_response
class Meta:
collection_name = "ctf"
ignore_unknown_fields = True
indexes = [
IndexModel([('guild_id', 1), ('name', 1)], unique=True),
IndexModel([('guild_id', 1), ('category_id', 1)], unique=True)
]
class FacebookFieldsModel(EmbeddedMongoModel):
public_fields = fields.CharField()
email = fields.EmailField()
birthday = fields.DateTimeField()
user_photo = fields.URLField() # "Change to ImageField"
token = fields.CharField()