class Article(Document):
type = TextField(default='article')
publisher = TextField()
publisher_location = TextField()
published_date = DateTimeField()
title = TextField()
link = TextField()
image_link = TextField()
image_type = TextField()
source = TextField()
raw = TextField()
word_frequencies = ListField(DictField(Mapping.build(
word = TextField(),
frequency = IntegerField()
)))
entity_frequencies = ListField(DictField(Mapping.build(
entity_type = TextField(),
entity = TextField(),
frequency = IntegerField()
)))
def extract_word_frequencies(self, tokens):
words = [w.lower() for sent in tokens for w in sent if w.isalpha() and w not in stopwords.words('english')]
fdwords = nltk.FreqDist(words)
self.fdwords = fdwords
for w in self.fdwords.keys():
frequency = self.fdwords[w]
self.word_frequencies.append(word=w, frequency=frequency)
def extract_entity_frequencies(self, tokens):
pos_tagged_tokens = [nltk.pos_tag(t) for t in tokens]
loc = LocationChunker()
trees = batch_ne_chunk(pos_tagged_tokens)
entity_types = ['PERSON', 'ORGANIZATION', 'GPE', 'LOCATION', 'FACILITY']
for entity_type in entity_types:
entity_freq_dict = {}
chunks = [sub_leaves(t, entity_type) for t in trees]
for sent in chunks:
for c in sent:
entity = ' '.join([w[0] for w in c])
entity_freq_dict[entity] = entity_freq_dict.get(entity, 0) + 1
# A secondary attempt at extracting locations based on reference
# to lists of place names
if entity_type == 'LOCATION':
for sent in pos_tagged_tokens:
t = loc.parse(sent)
chunks = sub_leaves(t, 'LOCATION')
for c in chunks:
entity = ' '.join([w[0] for w in c])
entity_freq_dict[entity] = entity_freq_dict.get(entity, 0) + 1
entity_freq_list = [(entity_freq_dict[e], e) for e in entity_freq_dict.keys()]
entity_freq_list.sort(reverse=True)
for e in entity_freq_list:
self.entity_frequencies.append(
entity_type=entity_type.lower(),
entity=e[1],
frequency=e[0])
# Basic entity extraction, but not precise enough so using
# above approach instead
# def extract_entity_frequencies(self, tokens):
# pos_tagged_tokens = [nltk.pos_tag(t) for t in tokens]
#
# # Flatten the list since we're not using sentence structure
# # and sentences are guaranteed to be separated by a special
# # POS tuple such as ('.', '.')
#
# pos_tagged_tokens = [token for sent in pos_tagged_tokens for token in sent]
#
# all_entity_chunks = []
# previous_pos = None
# current_entity_chunk = []
# for (token, pos) in pos_tagged_tokens:
# if pos == previous_pos and pos.startswith('NN'):
# current_entity_chunk.append(token)
# elif pos.startswith('NN'):
# if current_entity_chunk != []:
#
# # Note that current_entity_chunk could be a duplicate when appended,
# # so frequency analysis again becomes a consideration
#
# all_entity_chunks.append((' '.join(current_entity_chunk), pos))
# current_entity_chunk = [token]
# previous_pos = pos
#
# # Store the chunks as an index for the article
# # and account for frequency while we're at it
#
# entity_freq_dict = {}
# for c in all_entity_chunks:
# entity_freq_dict[c[0]] = entity_freq_dict.get(c[0], 0) + 1
# entity_freq_list = [(entity_freq_dict[e], e) for e in entity_freq_dict.keys()]
# entity_freq_list.sort(reverse=True)
# for e in entity_freq_list:
# self.entity_frequencies.append(entity=e[1], frequency=e[0])
def get_content(self):
return self.raw
def set_content(self, content):
self.raw = content
if content and len(content) > 0:
sentences = nltk.sent_tokenize(content)
tokens = [nltk.word_tokenize(s) for s in sentences]
self.extract_word_frequencies(tokens)
self.extract_entity_frequencies(tokens)
content = property(get_content, set_content)
def __str__(self):
print self.title
class Page(Document):
type = TextField(default="page")
url = TextField()
raw = TextField()
content = TextField()
links = ListField(TextField())
rank = FloatField(default=0)
last_checked = DateTimeField(default=datetime.now)
def is_valid(self):
return (datetime.now() - self.last_checked).days < 7
def update(self):
print("updating page")
parse = urlparse(self.url)
robotstxt = RobotsTxt.get_by_domain(parse.scheme, parse.netloc)
#if not robotstxt.is_allowed(self.url):
# return False
while cache.get(parse.netloc) is not None:
time.sleep(1)
cache.set(parse.netloc, True, 10)
print("getting: {}".format(self.url))
resp = requests.get(self.url, headers={'User-Agent':
settings.USER_AGENT})
ctype = resp.headers['content-type']
if not ctype.startswith("text/html"):
print("unsupported content-type: {}".format(ctype))
return
print("setting Page.content...")
self.content = resp.text
self.raw = resp.text
self.last_checked = datetime.now()
self.store(settings.db)
@staticmethod
def count():
r = settings.db.view("page/by_url", limit=0)
return r.total_rows
@staticmethod
def get_top_by_rank(limit=10):
r = settings.db.view("page/by_rank", limit=limit)
docs = []
for row in r.rows:
docs.append(Page.load(settings.db, row.value))
return docs
@staticmethod
def get_by_url(url, update=True):
r = settings.db.view("page/by_url", key=url)
if len(r.rows) == 1:
doc = Page.load(settings.db, r.rows[0].value)
if doc.is_valid():
return doc
elif not update:
return None
else:
doc = Page(url=url)
print("Page.get_by_url: doc.update() ...")
doc.update()
return doc
@staticmethod
def get_id_by_url(url, update=True):
r = settings.db.view("page/by_url", key=url)
if len(r) == 1:
return r.rows[0].value
else:
doc = Page.get_by_url(url, update=update)
if doc is not None:
return doc.id
else:
return None
@staticmethod
def get_links_to_url(url):
return [row.value for row in settings.db.view("page/links_to_url", key=url).rows]
class EntityFormModelDocument(FormModelDocument):
entity_type = ListField(TextField())
def __init__(self, id=None):
super(EntityFormModelDocument, self).__init__(id)
class UserPermissionDocument(DocumentBase):
user_id = IntegerField()
project_ids = ListField(TextField())
def __init__(self):
DocumentBase.__init__(self, document_type='UserPermission')
class Project(DocumentBase):
name = TextField()
goals = TextField()
project_type = TextField()
entity_type = TextField()
activity_report = TextField()
devices = ListField(TextField())
qid = TextField()
state = TextField()
sender_group = TextField()
reminder_and_deadline = DictField()
data_senders = ListField(TextField())
language = TextField(default='en')
def __init__(self,
id=None,
name=None,
goals=None,
project_type=None,
entity_type=None,
devices=None,
state=ProjectState.INACTIVE,
activity_report=None,
sender_group=None,
language='en'):
assert entity_type is None or is_string(
entity_type), "Entity type %s should be a string." % (
entity_type, )
DocumentBase.__init__(self, id=id, document_type='Project')
self.devices = []
self.name = name.lower() if name is not None else None
self.goals = goals
self.project_type = project_type
self.entity_type = entity_type
self.devices = devices
self.state = state
self.activity_report = activity_report
self.sender_group = sender_group
self.reminder_and_deadline = {
"deadline_type": "Following",
"should_send_reminder_to_all_ds": False,
"has_deadline": True,
"deadline_month": "5",
"frequency_period": "month"
}
self.language = language
def is_activity_report(self):
return self.activity_report == "yes"
def get_data_senders(self, dbm):
all_data, fields, label = load_all_subjects_of_type(dbm, type=REPORTER)
return [
dict(zip(fields, data["cols"])) for data in all_data
if data['short_code'] in self.data_senders
]
def _get_data_senders_ids_who_made_submission_for(self, dbm,
deadline_date):
start_date, end_date = self.deadline(
).get_applicable_frequency_period_for(deadline_date)
form_code = self._load_form(dbm).form_code
data_senders_with_submission = get_reporters_who_submitted_data_for_frequency_period(
dbm, form_code, start_date, end_date)
return [ds.short_code for ds in data_senders_with_submission]
def get_data_senders_without_submissions_for(self, deadline_date, dbm):
data_sender_ids_with_submission = self._get_data_senders_ids_who_made_submission_for(
dbm, deadline_date)
all_data_senders = self.get_data_senders(dbm)
data_senders_without_submission = [
data_sender for data_sender in all_data_senders
if data_sender['short_code'] not in data_sender_ids_with_submission
]
return data_senders_without_submission
def deadline(self):
return Deadline(self._frequency(), self._deadline_type())
def _frequency(self):
if self.reminder_and_deadline.get('frequency_period') == 'month':
return Month(int(self.reminder_and_deadline.get('deadline_month')))
if self.reminder_and_deadline.get('frequency_period') == 'week':
return Week(int(self.reminder_and_deadline.get('deadline_week')))
def has_deadline(self):
return self.reminder_and_deadline.get('has_deadline')
def _deadline_type(self):
return self.reminder_and_deadline.get('deadline_type')
def _frequency_period(self):
return self.reminder_and_deadline.get('frequency_period')
def get_deadline_day(self):
if self.reminder_and_deadline.get('frequency_period') == 'month':
return int(self.reminder_and_deadline.get('deadline_month'))
def should_send_reminders(self, as_of, days_relative_to_deadline):
next_deadline_day = self.deadline().current(as_of)
if next_deadline_day is not None:
if as_of == next_deadline_day + timedelta(
days=days_relative_to_deadline):
return True
return False
def _check_if_project_name_unique(self, dbm):
rows = dbm.load_all_rows_in_view('project_names', key=self.name)
if len(rows) and rows[0]['value'] != self.id:
raise DataObjectAlreadyExists('Project', "Name",
"'%s'" % self.name)
def save(self, dbm):
assert isinstance(dbm, DatabaseManager)
self._check_if_project_name_unique(dbm)
return dbm._save_document(self)
def update(self, value_dict):
attribute_list = [item[0] for item in (self.items())]
for key in value_dict:
if key in attribute_list:
setattr(
self, key,
value_dict.get(key).lower()) if key == 'name' else setattr(
self, key, value_dict.get(key))
def update_questionnaire(self, dbm):
form_model = self._load_form(dbm)
form_model.name = self.name
form_model.activeLanguages = [self.language]
form_model.entity_type = [self.entity_type] if is_string(
self.entity_type) else self.entity_type
form_model.save()
def activate(self, dbm):
form_model = self._load_form(dbm)
form_model.activate()
form_model.save()
self.state = ProjectState.ACTIVE
self.save(dbm)
def deactivate(self, dbm):
form_model = self._load_form(dbm)
form_model.deactivate()
form_model.save()
self.state = ProjectState.INACTIVE
self.save(dbm)
def to_test_mode(self, dbm):
form_model = self._load_form(dbm)
form_model.set_test_mode()
form_model.save()
self.state = ProjectState.TEST
self.save(dbm)
def delete(self, dbm):
if self.id is not None:
dbm.database.delete(self)
#The method name sucks but until we make Project DataObject we can't make the method name 'void'
def set_void(self, dbm, void=True):
self.void = void
self.save(dbm)
def is_on_type(self, type):
return self.entity_type == type
def _load_form(self, dbm):
form_model = dbm.get(self.qid, FormModel)
return form_model
def delete_datasender(self, dbm, entity_id):
self.data_senders.remove(entity_id)
self.save(dbm)
def associate_data_sender_to_project(self, dbm, data_sender_code):
self.data_senders.append(data_sender_code)
self.save(dbm)
class ContactDocument(DocumentBase):
"""
The couch entity document. It abstracts out the couch related functionality and inherits from the Document class
of couchdb-python.
A schema for the entity is enforced here.
"""
aggregation_paths = DictField()
geometry = DictField()
centroid = ListField(FloatField())
gr_id = TextField()
short_code = TextField()
data = DictField()
groups = ListField(TextField())
custom_groups = ListField(TextField())
def __init__(self,
id=None,
aggregation_paths=None,
geometry=None,
centroid=None,
gr_id=None,
short_code=None):
DocumentBase.__init__(self, id=id, document_type='Contact')
self.aggregation_paths = (aggregation_paths
if aggregation_paths is not None else {})
self._geometry = geometry
self._centroid = centroid
self._gr_id = gr_id
self.short_code = short_code
@property
def entity_type(self):
if attributes.TYPE_PATH in self.aggregation_paths:
return self.aggregation_paths[attributes.TYPE_PATH]
else:
return None
@property
def email(self):
if self.data.get('email'):
return self.data['email']['value']
return None
@property
def name(self):
if self.data.get('name'):
return self.data.get('name')['value']
return None
@entity_type.setter
def entity_type(self, typ):
self.aggregation_paths[attributes.TYPE_PATH] = typ
@property
def location(self):
if attributes.GEO_PATH in self.aggregation_paths:
return self.aggregation_paths[attributes.GEO_PATH]
else:
return None
@location.setter
def location(self, loc):
self.aggregation_paths[attributes.GEO_PATH] = loc
def add_group(self, group_name):
self.groups.append(group_name)
def add_custom_group(self, group_name):
if not 'custom_groups' in self._data:
self._data['custom_groups'] = []
if group_name not in self.custom_groups:
self.custom_groups.append(group_name)
def remove_custom_group(self, group_name):
if group_name in self.custom_groups:
self.custom_groups.remove(group_name)
class Case(Document):
_id = TextField()
type = TextField(default="case")
status = TextField()
assignee = TextField()
description = TextField()
subject = TextField()
creator_email = TextField()
creator_useragent = TextField()
creator_name = TextField()
creator_username = TextField()
url = TextField()
created = DateTimeField()
history = ListField(
DictField(
Mapping.build(at=DateTimeField(), by=TextField(),
text=TextField())))
def __repr__(self):
return "<Case ('%s')>" % self._id
def change_status(self, new_status, by):
self.status = new_status
self.store(self.db)
def reassign(self, new_assignee, by, text=""):
self.assignee = new_assignee
entry = dict(by=by,
at=datetime.datetime.utcnow(),
text="Case reassigned to '%s'\n\n%s" %
(new_assignee, text))
self.history.append(entry)
self.store(self.db)
def add_worklog_entry(self, by, text):
entry = dict(by=by, at=datetime.datetime.utcnow(), text=text)
self.history.append(entry)
self.store(self.db)
# Override base class members to hold the database connection
@classmethod
def load(cls, db, id):
ret = super(Case, cls).load(db, id)
if not ret:
raise InvalidCase("No case with id %s" % id)
ret.db = db
return ret
def store(self, db):
super(Case, self).store(db)
self.db = db
@property
def last_modified(self):
return self.history[-1].at
@property
def caseid(self):
return self._id
@property
def caseno(self):
"Returns case number"
return int(self._id.replace("case-", ""))
@classmethod
def new(cls, **kargs):
ret = cls(**kargs)
item = dict(at=ret.created,
by=ret.creator_name or ret.creator_email,
text="Case created")
ret.history.append(item)
return ret
@classmethod
def all(cls, db, typ="all", sort="status", desc="false", staleok=False):
view = {
"created": "cases/sort-created",
"caseid": "cases/sort-caseid",
"assigned": "cases/sort-assignee",
"user": "******",
"lastmodified": "cases/sort-lastmodified",
"status": "cases/sort-status",
"subject": "cases/sort-subject",
"notes": "cases/sort-numnotes"
}[sort]
if sort == "status":
extra = dict(reduce=False, descending=desc)
else:
extra = dict(descending=desc)
if staleok:
extra['stale'] = "ok"
if typ == "all":
view = view.replace("-", "-all-")
result = cls.view(db, view, include_docs=True, **extra)
return result.rows
elif typ == "new":
startkey, endkey = (["new"], ["replied"])
elif typ == "closed":
startkey, endkey = (["closed"], ["new"])
elif typ == "replied":
startkey, endkey = (["replied"], False)
else:
raise KeyError("No such case type '%s'" % typ)
if desc == "true":
startkey, endkey = endkey, startkey
if startkey:
extra['startkey'] = startkey
if endkey:
extra['endkey'] = endkey
result = cls.view(db, view, include_docs=True, **extra)
return result.rows
def __eq__(self, second):
return self._id == second._id
class HomeState(Document):
home_id = TextField()
type = TextField(default="home_state")
name = TextField()
time = DateTimeField()
periodic_update = BooleanField()
devices = ListField(DictField())
# Model 1
def feature_vector(self):
feature_vector = []
# Ensure all times are in UTC when creating feature vectors
utctime = datetime.utcfromtimestamp(self.time.timestamp())
# Append with day of week
feature_vector.append(utctime.weekday())
# Append with hour and minute number in 10 minutes interval
feature_vector.append(utctime.hour)
feature_vector.append(int(utctime.minute / 10))
# Loop over all devices sorted by ID - this will ensure the same order of features for every feature vector
for device in sorted(self.devices, key=lambda d: d['device_id']):
device_feature_vector = self.feature_vector_device(device)
if device_feature_vector != None:
feature_vector.extend(device_feature_vector)
return feature_vector
# Model 2
def feature_vector_for_output_device(self, output_device):
feature_vector = []
# Ensure all times are in UTC when creating feature vectors
utctime = datetime.utcfromtimestamp(self.time.timestamp())
# Append with day of week
feature_vector.append(utctime.weekday())
# Append with hour and minute number in 15 minutes interval
feature_vector.append(utctime.hour)
feature_vector.append(utctime.minute / 15)
# Loop over all devices sorted by ID - this will ensure the same order of features for every feature vector
for device in sorted(self.devices, key=lambda d: d['device_id']):
if output_device['device_id'] != device['device_id']:
device_feature_vector = self.feature_vector_device(device)
if device_feature_vector != None:
feature_vector.extend(device_feature_vector)
return feature_vector
# Model 3
def feature_vector_for_output_device_in_same_location(self, output_device):
feature_vector = []
# Ensure all times are in UTC when creating feature vectors
utctime = datetime.utcfromtimestamp(self.time.timestamp())
# Append with day of week
feature_vector.append(utctime.weekday())
# Append with hour and minute number in 15 minutes interval
feature_vector.append(utctime.hour)
feature_vector.append(utctime.minute / 15)
# Loop over all devices sorted by ID - this will ensure the same order of features for every feature vector
for device in sorted(self.devices, key=lambda d: d['device_id']):
print(device)
if output_device['device_id'] != device[
'device_id'] and output_device['location'] == device[
'location']:
device_feature_vector = self.feature_vector_device(device)
if device_feature_vector != None:
feature_vector.extend(device_feature_vector)
return feature_vector
def output_vector(self):
output_vector = []
# Loop over all devices sorted by ID - this will ensure the same order of features for every output vector
for device in sorted(self.devices, key=lambda d: d['device_id']):
device_output_vector = self.output_vector_device(device)
if device_output_vector != None:
output_vector.extend(device_output_vector)
return output_vector
def output_devices(self):
return [
device for device in self.devices
if device['type'] in ('BinaryPowerSwitchDevice', 'DimmerDevice')
]
def output_vector_device(self, device):
if device['type'] == 'DimmerDevice':
return [device['state']]
elif device['type'] == 'BinaryPowerSwitchDevice':
return [1 if device['state'] > 1 else 0]
else: # Other devices does not generate a feature vector
return None
def output_vector_for_device_id(self, device_id):
# Find output device based on id
device = next(device for device in self.devices
if device['device_id'] == device_id)
if device['type'] == 'DimmerDevice':
return device['state']
elif device['type'] == 'BinaryPowerSwitchDevice':
return 1 if device['state'] > 1 else 0
else: # Other devices does not generate a feature vector
return None
def feature_vector_device(self, device):
if device['type'] == 'BinarySensorDevice':
return [1 if device['state'] > 1 else 0]
elif device['type'] == 'MultiSensorDevice':
return [1 if device['state'] > 1 else 0, device['lux'] / 2]
elif device['type'] == 'BinaryPowerSwitchDevice':
return [device['power_state']]
else: # Other devices does not generate a feature vector
return None
class Container(Document):
name = TextField()
container_type = TextField()
size = DictField(Mapping.build(
amount = DecimalField(),
unit = TextField()
))
contents = ListField(DictField(Mapping.build(
amount = DecimalField(),
unit = TextField(),
content = TextField(),
updated_datetime = DateTimeField(default=datetime.now),
temperature = DictField(Mapping.build(
degrees = DecimalField(),
unit = TextField()
)),
content_type = TextField()
)))
full = BooleanField(default=False)
def total_filled(self):
"""Return the amount of filled content. However, there is no weight to volume conversion.
E.g. weight is free in volume-organized container
volume is free in weight-organized containers
"""
total_amount = 0
target_unit = self.size.unit
for content in self.contents:
if content.unit in VOLUME_UNITS and target_unit in WEIGHT_UNITS:
continue
converted_amount = convert_amount( (content.amount, content.unit), target_unit)
total_amount += converted_amount
return (total_amount, target_unit)
def heat_contents(self, temp_tuple):
"""Set all the contents of the container to be a temperature of temp_tuple
temp_tuple is (amount, units), e.g. (70, 'C')
"""
if temp_tuple[1].upper() not in TEMP_UNITS:
raise ContainerError("Unable to use the specified temperature unit %s" % (temp_tuple[1]))
else:
for content in self.contents:
content.temperature = dict( degrees = temp_tuple[0],
unit = temp_tuple[1].upper()
)
def set_size(self, size_tuple):
if not ((size_tuple[1].lower() in WEIGHT_UNITS) or (size_tuple[1].lower() in VOLUME_UNITS)):
raise ContainerError("Unable to use the specified unit: %s" % (size_tuple[1]))
self.size = {'amount': size_tuple[0],
'unit': size_tuple[1].lower() }
def determine_content_type(self, content):
if "wort" in content.lower():
return ContentType.Wort
elif "hops" in content.lower():
return ContentType.Hops
elif "yeast" in content.lower():
return ContentType.Yeast
elif "malt" in content.lower():
return ContentType.Malt
elif "water" in content.lower():
return ContentType.Water
elif "beer" in content.lower():
return ContentType.Beer
def add_content(self, content, amount, temp = (23, "C"), content_type=None, updated_datetime=None):
"""
:param amount - may be dictionary: {"amount": amount, "unit": unit}, or tuple (amount, unit)
"""
if not updated_datetime:
updated_datetime=datetime.now()
LOGGER.info("Attempting to add content %s (amount %s, temp %s)", content, amount, temp)
amount_dict = {}
temp_dict = {}
if not content_type:
content_type = self.determine_content_type(content)
if "unit" not in amount:
amount_dict = {"amount": amount[0],
"unit": amount[1]}
else:
amount_dict = amount
if amount_dict["unit"] != self.size.unit and amount_dict["unit"] in VOLUME_UNITS:
raise ContainerError("Only able to add volume content measured in '{0}'".format(self.size.unit))
#Assume room temperature
if "unit" not in temp:
temp_dict = {"degrees": 23,
"unit": "C"}
else:
temp_dict = temp
if not amount_dict:
raise ContainerError("Amount must be provided as (amount, unit) or as dictinoary ")
# adding_amount is the value to add to the container
adding_amount = amount_dict["amount"]
if amount_dict["unit"] != self.size.unit:
try:
adding_amount = convert_amount(amount_dict, self.size.unit)
except ContainerError as e:
LOGGER.info("Unable to convert to unit %s. Not adding any volume. %s", self.size.unit, e)
adding_amount = 0
filled = self.total_filled()
if float(filled[0]) + float(adding_amount) > self.size.amount:
adding_amount = self.size.amount - filled[0]
self.contents.append(
dict( amount = adding_amount,
unit = self.size.unit,
content = content,
temperature = temp_dict,
content_type = content_type,
updated_datetime = updated_datetime,
)
)
raise ContainerError("Error filling container: trying to add too much, only %s %s added (%s %s not added)" % (adding_amount, self.size.unit, amount_dict["amount"] - adding_amount, self.size.unit))
LOGGER.info("Adding full content %s %s %s" % (content, amount_dict["amount"], amount_dict["unit"]))
self.contents.append(
dict( amount = amount_dict["amount"],
unit = amount_dict["unit"],
content = content,
temperature = temp_dict,
content_type = content_type,
updated_datetime = updated_datetime,
))
if self.total_filled()[0] >= self.size.amount:
self.full = True
def remove_all(self):
"""Remove all contents, and return a list
"""
removed = []
for content in self.contents:
removed.extend(self.remove_content(content.content, (content.amount, content.unit)))
return removed
def add_all(self, content_list):
"""content_list is a list of tuples ('content',(amount, unit), (degrees, unit))
add the specified content to the list
"""
LOGGER.info("Adding contents to container: %s", content_list)
if not content_list:
LOGGER.info("No contents to add")
return
for content in content_list:
LOGGER.info("Attempting to add content %s", content)
#for (content, amount_tuple, temperature_tuple) in content_list:
try:
self.add_content(content[0], content[1], content[2])
LOGGER.info("Added content: %s", content)
except IndexError:
LOGGER.error("Unable to add content %s", content)
def remove_content(self, content, amount_tuple):
"""Remove the specified amount of content
:return a list of tuples of the removed contents (content, amount, temperature)
"""
if content not in [ c.content for c in self.contents ]:
raise ContainerError("Unable to remove the specify contents %s. The container is not currently filled with any")
same_content = [ c for c in self.contents if c.content.lower() == content ]
reducing_amount = amount_tuple[0]
reducing_unit = amount_tuple[1]
all_removed = []
for current_content in self.contents:
if current_content.content.lower() == content.lower():
reducing_amount = convert_amount( (reducing_amount, reducing_unit), current_content.unit)
LOGGER.info("Converted amount to remove: %s %s", reducing_amount, current_content.unit)
if reducing_amount > current_content.amount:
LOGGER.info("Amount reduce %s %s more than current contained content %s %s", reducing_amount, current_content.unit, current_content.amount, current_content.unit)
reducing_amount = current_content.amount
content_tuple = (content,
(reducing_amount, current_content.unit),
(current_content.temperature.degrees, current_content.temperature.unit))
LOGGER.info("Removed content Appending content, amount, temperature %s to removed list %s", content_tuple, all_removed)
# Reduce the total amount of content to be removed
reducing_amount -= current_content.amount
all_removed.append(content_tuple)
current_content.amount = 0
else:
# Remove everything specified
LOGGER.info("Removing full specified amount %s %s", reducing_amount, current_content.unit)
current_content.amount -= reducing_amount
content_tuple = (content,
{"amount": reducing_amount,
"unit": current_content.unit},
{"degrees": current_content.temperature.degrees,
"unit": current_content.temperature.unit})
LOGGER.info("Updated content amount after reduction: %s %s", current_content.amount, current_content.unit)
#LOGGER.info("Removed list: appending content, amount, temperature %s to removed list %s", content_tuple, all_removed)
LOGGER.info("Removed list addition: %s", content_tuple)
all_removed.append(content_tuple)
LOGGER.info("List of all removed contents: %s", all_removed)
# Remove the empty contents
self.contents = [ c for c in self.contents if c.amount ]
if self.total_filled()[0] < self.size.amount:
self.full = False
LOGGER.info("All removed content: %s", all_removed)
return all_removed
class MyRingUser(Document):
_id = TextField()
#firstname = TextField() DEPRECATED! Use "name" for firstname if needed
name = TextField()
lastname = TextField()
email = TextField()
billingemail = TextField()
is_org = BooleanField()
location = TextField()
url = TextField()
profilepic = TextField()
about = TextField()
#people = DictField()
onlogin = TextField()
people = ListField(
DictField(
Mapping.build(handle=TextField(),
addedby=TextField(),
added=DateTimeField(default=datetime.now))))
teams = ListField(
DictField(
Mapping.build(
teamname=TextField(),
members=ListField(
DictField(
Mapping.build(
handle=TextField(),
addedby=TextField(),
added=DateTimeField(default=datetime.now)))),
rings=ListField(
DictField(
Mapping.build(
handle=TextField(),
ringname=TextField(),
addedby=TextField(),
added=DateTimeField(default=datetime.now)))),
roles=ListField(
DictField(
Mapping.build(
role=TextField(),
addedby=TextField(),
added=DateTimeField(default=datetime.now)))),
addedby=TextField(),
added=DateTimeField(default=datetime.now))))
salt = TextField()
passhash = TextField()
guid = TextField()
added = DateTimeField(default=datetime.now)
branchof = TextField()
rings = ListField(
DictField(
Mapping.build(ringname=TextField(),
version=TextField(),
count=IntegerField(),
added=DateTimeField(default=datetime.now),
roles=ListField(
DictField(
Mapping.build(role=TextField(),
users=ListField(
Mapping.build())))))))
collections = ListField(
DictField(
Mapping.build(collectionname=TextField(),
collectiondescription=TextField(),
version=TextField(),
added=DateTimeField(default=datetime.now),
rings=ListField(
DictField(
Mapping.build(handle=TextField(),
ringname=TextField(),
version=TextField(),
layer=IntegerField()))),
roles=ListField(
DictField(
Mapping.build(role=TextField(),
users=ListField(
Mapping.build())))))))
badges = ListField(
DictField(
Mapping.build(badgename=TextField(),
badgedescription=TextField(),
added=DateTimeField(default=datetime.now))))
campaigns = ListField(
DictField(
Mapping.build(campaignname=TextField(),
campaigndescription=TextField(),
added=DateTimeField(default=datetime.now))))
is_active = BooleanField(default=True)
is_authenticated = BooleanField(default=False)
new_password_key = TextField()
new_password_requested = DateTimeField()
new_email = TextField()
new_email_key = TextField()
last_login = DateTimeField()
last_ip = TextField()
modified = DateTimeField(default=datetime.now)
class Group(Permission):
permissions = ListField(TextField)
class Page(Document):
type = TextField(default="page")
url = TextField()
content = TextField()
links = ListField(TextField())
rank = FloatField(default=0)
last_checked = DateTimeField(default=datetime.now)
def is_valid(self):
return (datetime.now() - self.last_checked).days < 7
def update(self):
parse = urlparse(self.url)
robotstxt = RobotsTxt.get_by_domain(parse.scheme, parse.netloc)
if not robotstxt.is_allowed(self.url):
return False
while cache.get(parse.netloc) is not None:
time.sleep(1)
cache.set(parse.netloc, True, 10)
print "getting", self.url
req = Request(self.url, None, {"User-Agent": settings.USER_AGENT})
resp = urlopen(req)
if not resp.info()["Content-Type"].startswith("text/html"):
return
self.content = resp.read().decode("utf8")
self.last_checked = datetime.now()
self.store(settings.db)
@staticmethod
def count():
r = settings.db.view("page/by_url", limit=0)
return r.total_rows
@staticmethod
def get_top_by_rank(limit=10):
r = settings.db.view("page/by_rank", limit=limit)
docs = []
for row in r.rows:
docs.append(Page.load(settings.db, row.value))
return docs
@staticmethod
def get_by_url(url, update=True):
r = settings.db.view("page/by_url", key=url)
if len(r.rows) == 1:
doc = Page.load(settings.db, r.rows[0].value)
if doc.is_valid():
return doc
elif not update:
return None
else:
doc = Page(url=url)
doc.update()
return doc
@staticmethod
def get_id_by_url(url, update=True):
r = settings.db.view("page/by_url", key=url)
if len(r) == 1:
return r.rows[0].value
else:
doc = Page.get_by_url(url, update=update)
if doc is not None:
return doc.id
else:
return None
@staticmethod
def get_links_to_url(url):
return [
row.value
for row in settings.db.view("page/links_to_url", key=url).rows
]
class Case(Document):
_id = TextField()
type = TextField(default="case")
status = TextField()
assignee = TextField()
description = TextField()
subject = TextField()
creator_email = TextField()
creator_useragent = TextField()
creator_name = TextField()
url = TextField()
created = DateTimeField()
history = ListField(
DictField(
Mapping.build(at=DateTimeField(), by=TextField(),
text=TextField())))
def __init__(self, **kargs):
super(Case, self).__init__(**kargs)
item = dict(at=self.created,
by=self.creator_name or self.creator_email,
text="Case created")
self.history.append(item)
def change_status(self, new_status, by):
self.status = new_status
entry = dict(by=by,
at=datetime.datetime.utcnow(),
text="Case status changed to '%s'" % new_status)
self.history.append(entry)
self.store(self.db)
def reassign(self, new_assignee, by):
self.assignee = new_assignee
entry = dict(by=by,
at=datetime.datetime.utcnow(),
text="Case reassigned to '%s'" % new_assignee)
self.history.append(entry)
self.store(self.db)
def add_worklog_entry(self, by, text):
entry = dict(by=by, at=datetime.datetime.utcnow(), text=text)
self.history.append(entry)
self.store(self.db)
# Override base class members to hold the database connection
@classmethod
def load(cls, db, id):
ret = super(Case, cls).load(db, id)
if not ret:
raise InvalidCase("No case with id %s" % id)
ret.db = db
return ret
def store(self, db):
super(Case, self).store(db)
self.db = db
@property
def last_modified(self):
return self.history[-1].at
@property
def caseid(self):
return self._id
@property
def caseno(self):
"Returns case number"
return int(self._id.replace("case-", ""))
@ViewField.define('cases')
def all(self, doc):
if doc.get("type", "") == "case":
yield doc["_id"], doc
def __eq__(self, second):
return self._id == second._id