def _from_son(cls, son, _auto_dereference=True, only_fields=None, created=False):
"""Create an instance of a Document (subclass) from a PyMongo SON.
"""
if not only_fields:
only_fields = []
# get the class name from the document, falling back to the given
# class if unavailable
class_name = son.get('_cls', cls._class_name)
data = dict(("%s" % key, value) for key, value in son.iteritems())
# Return correct subclass for document type
if class_name != cls._class_name:
cls = get_document(class_name)
changed_fields = []
errors_dict = {}
fields = cls._fields
if not _auto_dereference:
fields = copy.copy(fields)
for field_name, field in fields.iteritems():
field._auto_dereference = _auto_dereference
if field.db_field in data:
value = data[field.db_field]
try:
data[field_name] = (value if value is None
else field.to_python(value))
if field_name != field.db_field:
del data[field.db_field]
except (AttributeError, ValueError), e:
errors_dict[field_name] = e
def _from_son(cls, son, _auto_dereference=True, only_fields=None, created=False):
"""Create an instance of a Document (subclass) from a PyMongo SON.
"""
if not only_fields:
only_fields = []
# get the class name from the document, falling back to the given
# class if unavailable
class_name = son.get('_cls', cls._class_name)
data = dict(("%s" % key, value) for key, value in son.iteritems())
# Return correct subclass for document type
if class_name != cls._class_name:
cls = get_document(class_name)
changed_fields = []
errors_dict = {}
fields = cls._fields
if not _auto_dereference:
fields = copy.copy(fields)
for field_name, field in fields.iteritems():
field._auto_dereference = _auto_dereference
if field.db_field in data:
value = data[field.db_field]
try:
data[field_name] = (value if value is None
else field.to_python(value))
if field_name != field.db_field:
del data[field.db_field]
except (AttributeError, ValueError), e:
errors_dict[field_name] = e
def __expand_dynamic_values(self, name, value):
"""Expand any dynamic values to their correct types / values."""
if not isinstance(value, (dict, list, tuple)):
return value
# If the value is a dict with '_cls' in it, turn it into a document
is_dict = isinstance(value, dict)
if is_dict and '_cls' in value:
cls = get_document(value['_cls'])
return cls(**value)
if is_dict:
value = {
k: self.__expand_dynamic_values(k, v)
for k, v in value.items()
}
else:
value = [self.__expand_dynamic_values(name, v) for v in value]
# Convert lists / values so we can watch for any changes on them
EmbeddedDocumentListField = _import_class('EmbeddedDocumentListField')
if (isinstance(value, (list, tuple)) and
not isinstance(value, BaseList)):
if issubclass(type(self), EmbeddedDocumentListField):
value = EmbeddedDocumentList(value, self, name)
else:
value = BaseList(value, self, name)
elif isinstance(value, dict) and not isinstance(value, BaseDict):
value = BaseDict(value, self, name)
return value
def get_collectionlist(choice): # Object is called to update in "addcollection" part colllist_obj = CollectionList.objects.first() colllist = colllist_obj["collectionlist"] if(choice == "info"): len_coll = [] len_unlabeled = [] len_labeled= [] for coll in colllist: model = get_document(coll) len_unlbld = model.objects(label__exists = False).count() len_lbld = model.objects(label__exists = True).count() len_coll.append(len_unlbld + len_lbld) len_unlabeled.append(len_unlbld) len_labeled.append(len_lbld) collectionlist = zip(colllist, len_coll, len_unlabeled, len_labeled) return collectionlist elif(choice == "update"): return colllist_obj, colllist
def __expand_dynamic_values(self, name, value):
"""expand any dynamic values to their correct types / values"""
if not isinstance(value, (dict, list, tuple)):
return value
is_list = False
if not hasattr(value, 'items'):
is_list = True
value = dict([(k, v) for k, v in enumerate(value)])
if not is_list and '_cls' in value:
cls = get_document(value['_cls'])
return cls(**value)
data = {}
for k, v in value.items():
key = name if is_list else k
data[k] = self.__expand_dynamic_values(key, v)
if is_list: # Convert back to a list
data_items = sorted(data.items(), key=operator.itemgetter(0))
value = [v for k, v in data_items]
else:
value = data
# Convert lists / values so we can watch for any changes on them
if (isinstance(value, (list, tuple))
and not isinstance(value, BaseList)):
value = BaseList(value, self, name)
elif isinstance(value, dict) and not isinstance(value, BaseDict):
value = BaseDict(value, self, name)
return value
def __expand_dynamic_values(self, name, value):
"""expand any dynamic values to their correct types / values"""
if not isinstance(value, (dict, list, tuple)):
return value
is_list = False
if not hasattr(value, 'items'):
is_list = True
value = dict([(k, v) for k, v in enumerate(value)])
if not is_list and '_cls' in value:
cls = get_document(value['_cls'])
return cls(**value)
data = {}
for k, v in value.items():
key = name if is_list else k
data[k] = self.__expand_dynamic_values(key, v)
if is_list: # Convert back to a list
data_items = sorted(data.items(), key=operator.itemgetter(0))
value = [v for k, v in data_items]
else:
value = data
# Convert lists / values so we can watch for any changes on them
if (isinstance(value, (list, tuple)) and
not isinstance(value, BaseList)):
value = BaseList(value, self, name)
elif isinstance(value, dict) and not isinstance(value, BaseDict):
value = BaseDict(value, self, name)
return value
def __expand_dynamic_values(self, name, value):
"""Expand any dynamic values to their correct types / values."""
if not isinstance(value, (dict, list, tuple)):
return value
# If the value is a dict with '_cls' in it, turn it into a document
is_dict = isinstance(value, dict)
if is_dict and '_cls' in value:
cls = get_document(value['_cls'])
return cls(**value)
if is_dict:
value = {
k: self.__expand_dynamic_values(k, v)
for k, v in value.items()
}
else:
value = [self.__expand_dynamic_values(name, v) for v in value]
# Convert lists / values so we can watch for any changes on them
EmbeddedDocumentListField = _import_class('EmbeddedDocumentListField')
if (isinstance(value, (list, tuple))
and not isinstance(value, BaseList)):
if issubclass(type(self), EmbeddedDocumentListField):
value = EmbeddedDocumentList(value, self, name)
else:
value = BaseList(value, self, name)
elif isinstance(value, dict) and not isinstance(value, BaseDict):
value = BaseDict(value, self, name)
return value
def create(self, validated_data):
class_name = validated_data.pop('_class_name', None)
if class_name:
cls = get_document(class_name)
else:
cls = self.Meta.model
return self.get_serializer(cls).create(validated_data)
def create(self, validated_data):
class_name = validated_data.pop('_class_name', None)
if class_name:
cls = get_document(class_name)
else:
cls = self.Meta.model
return self.get_serializer(cls).create(validated_data)
def post(self, request, collname):
if "confirmpass" in request.POST:
user_pass = request.POST['user_pass']
reset_pass = config.get('dataset', 'reset_pass')
if(user_pass == reset_pass):
client_address = get_client_ip(request)
logging.info('RESET : ' + client_address + ' requested to reset all labels on ' + collname )
backup_json(collname)
model = get_document(collname)
model.objects.update(unset__label=1)
logging.info('RESET : Reset done for all labels on ' + collname + ', by ' + client_address)
sbj = "Reset All Labels"
msg = sbj + '\n\nCollection : ' + collname + '\nIP address : ' + client_address + "\n\nThis mail sent to : ebasar"
send_mail(sbj, msg, "ebasar")
if HOSTNAME[:9] == "applejack":
msg2 = msg + ", hurrial"
send_mail(sbj, msg2, "hurrial")
send_mail(sbj, msg2, "ebasar")
else:
send_mail(sbj, msg, "ebasar")
confirmed = True
result = 'All labels are successfully removed from '
return render(request, 'confirmpass.html', {
'thing' : collname,
'confirmed' : confirmed,
'result' : result,
})
else:
confirmed = False
action = 'resetlabels'
event = 'reset all labels for '
denied_msg = "Wrong password. Please try again."
return render(request, 'confirmpass.html', {
'action': action,
'event' : event,
'thing' : collname,
'confirmed' : confirmed,
'denied_msg' : denied_msg,
})
def _from_son(cls, son, _auto_dereference=True, only_fields=None, created=False):
"""Create an instance of a Document (subclass) from a PyMongo SON.
"""
if not only_fields:
only_fields = []
# get the class name from the document, falling back to the given
# class if unavailable
class_name = son.get('_cls', cls._class_name)
data = dict(("%s" % key, value) for key, value in son.items())
# Return correct subclass for document type
if class_name != cls._class_name:
cls = get_document(class_name)
changed_fields = []
errors_dict = {}
fields = cls._fields
if not _auto_dereference:
fields = copy.copy(fields)
for field_name, field in fields.items():
field._auto_dereference = _auto_dereference
if field.db_field in data:
value = data[field.db_field]
try:
data[field_name] = (value if value is None
else field.to_python(value))
if field_name != field.db_field:
del data[field.db_field]
except (AttributeError, ValueError) as e:
errors_dict[field_name] = e
elif field.default:
default = field.default
if isinstance(default, collections.Callable):
default = default()
if isinstance(default, BaseDocument):
changed_fields.append(field_name)
elif not only_fields or field_name in only_fields:
changed_fields.append(field_name)
if errors_dict:
errors = "\n".join(["%s - %s" % (k, v)
for k, v in list(errors_dict.items())])
msg = ("Invalid data to create a `%s` instance.\n%s"
% (cls._class_name, errors))
raise InvalidDocumentError(msg)
if cls.STRICT:
data = dict((k, v)
for k, v in data.items() if k in cls._fields)
obj = cls(__auto_convert=False, _created=created, __only_fields=only_fields, **data)
obj._changed_fields = changed_fields
if not _auto_dereference:
obj._fields = fields
return obj
def _from_son(cls, son, _auto_dereference=True, only_fields=None, created=False):
"""Create an instance of a Document (subclass) from a PyMongo
SON.
"""
if not only_fields:
only_fields = []
if son and not isinstance(son, dict):
raise ValueError("The source SON object needs to be of type 'dict'")
# Get the class name from the document, falling back to the given
# class if unavailable
class_name = son.get('_cls', cls._class_name)
# Convert SON to a dict, making sure each key is a string
data = {str(key): value for key, value in son.iteritems()}
# Return correct subclass for document type
if class_name != cls._class_name:
cls = get_document(class_name)
changed_fields = []
errors_dict = {}
fields = cls._fields
if not _auto_dereference:
fields = copy.copy(fields)
for field_name, field in fields.iteritems():
field._auto_dereference = _auto_dereference
if field.db_field in data:
value = data[field.db_field]
try:
data[field_name] = (value if value is None
else field.to_python(value))
if field_name != field.db_field:
del data[field.db_field]
except (AttributeError, ValueError) as e:
errors_dict[field_name] = e
if errors_dict:
errors = '\n'.join(['%s - %s' % (k, v)
for k, v in errors_dict.items()])
msg = ('Invalid data to create a `%s` instance.\n%s'
% (cls._class_name, errors))
raise InvalidDocumentError(msg)
# In STRICT documents, remove any keys that aren't in cls._fields
if cls.STRICT:
data = {k: v for k, v in data.iteritems() if k in cls._fields}
obj = cls(__auto_convert=False, _created=created, __only_fields=only_fields, **data)
obj._changed_fields = changed_fields
if not _auto_dereference:
obj._fields = fields
return obj
def _from_son(cls, son, _auto_dereference=False):
# get the class name from the document, falling back to the given
# class if unavailable
class_name = son.get('_cls', cls._class_name)
# Return correct subclass for document type
if class_name != cls._class_name:
cls = get_document(class_name)
return cls(_son=son)
def _from_son(cls, son, _auto_dereference=False):
# get the class name from the document, falling back to the given
# class if unavailable
class_name = son.get('_cls', cls._class_name)
# Return correct subclass for document type
if class_name != cls._class_name:
cls = get_document(class_name)
return cls(_son=son)
def _from_son(cls, son, _auto_dereference=True):
"""Create an instance of a Document (subclass) from a PyMongo SON.
"""
# get the class name from the document, falling back to the given
# class if unavailable
class_name = son.get('_cls', cls._class_name)
data = dict(("%s" % key, value) for key, value in son.items())
if not UNICODE_KWARGS:
# python 2.6.4 and lower cannot handle unicode keys
# passed to class constructor example: cls(**data)
to_str_keys_recursive(data)
# Return correct subclass for document type
if class_name != cls._class_name:
cls = get_document(class_name)
changed_fields = []
errors_dict = {}
fields = cls._fields
if not _auto_dereference:
fields = copy.copy(fields)
for field_name, field in fields.items():
field._auto_dereference = _auto_dereference
if field.db_field in data:
value = data[field.db_field]
try:
data[field_name] = (value if value is None
else field.to_python(value))
if field_name != field.db_field:
del data[field.db_field]
except (AttributeError, ValueError) as e:
errors_dict[field_name] = e
elif field.default:
default = field.default
if isinstance(default, collections.Callable):
default = default()
if isinstance(default, BaseDocument):
changed_fields.append(field_name)
if errors_dict:
errors = "\n".join(["%s - %s" % (k, v)
for k, v in list(errors_dict.items())])
msg = ("Invalid data to create a `%s` instance.\n%s"
% (cls._class_name, errors))
raise InvalidDocumentError(msg)
obj = cls(__auto_convert=False, **data)
obj._changed_fields = changed_fields
obj._created = False
if not _auto_dereference:
obj._fields = fields
return obj
def _from_son(cls, son, _auto_dereference=True):
"""Create an instance of a Document (subclass) from a PyMongo SON.
"""
# get the class name from the document, falling back to the given
# class if unavailable
class_name = son.get('_cls', cls._class_name)
data = dict(("%s" % key, value) for key, value in iter(son.items()))
if not UNICODE_KWARGS:
# python 2.6.4 and lower cannot handle unicode keys
# passed to class constructor example: cls(**data)
to_str_keys_recursive(data)
# Return correct subclass for document type
if class_name != cls._class_name:
cls = get_document(class_name)
changed_fields = []
errors_dict = {}
fields = cls._fields
if not _auto_dereference:
fields = copy.copy(fields)
for field_name, field in fields.items():
field._auto_dereference = _auto_dereference
if field.db_field in data:
value = data[field.db_field]
try:
data[field_name] = (value if value is None else
field.to_python(value))
if field_name != field.db_field:
del data[field.db_field]
except (AttributeError, ValueError) as e:
errors_dict[field_name] = e
elif field.default:
default = field.default
if callable(default):
default = default()
if isinstance(default, BaseDocument):
changed_fields.append(field_name)
if errors_dict:
errors = "\n".join(
["%s - %s" % (k, v) for k, v in list(errors_dict.items())])
msg = ("Invalid data to create a `%s` instance.\n%s" %
(cls._class_name, errors))
raise InvalidDocumentError(msg)
obj = cls(__auto_convert=False, **data)
obj._changed_fields = changed_fields
obj._created = False
if not _auto_dereference:
obj._fields = fields
return obj
def _from_son(cls, son, _auto_dereference=True, only_fields=None, created=False):
"""Create an instance of a Document (subclass) from a PyMongo
SON.
"""
if not only_fields:
only_fields = []
# Get the class name from the document, falling back to the given
# class if unavailable
class_name = son.get('_cls', cls._class_name)
# Convert SON to a dict, making sure each key is a string
data = {str(key): value for key, value in son.items()}
# Return correct subclass for document type
if class_name != cls._class_name:
cls = get_document(class_name)
changed_fields = []
errors_dict = {}
fields = cls._fields
if not _auto_dereference:
fields = copy.copy(fields)
for field_name, field in fields.items():
field._auto_dereference = _auto_dereference
if field.db_field in data:
value = data[field.db_field]
try:
data[field_name] = (value if value is None
else field.to_python(value))
if field_name != field.db_field:
del data[field.db_field]
except (AttributeError, ValueError) as e:
errors_dict[field_name] = e
if errors_dict:
errors = '\n'.join(['%s - %s' % (k, v)
for k, v in list(errors_dict.items())])
msg = ('Invalid data to create a `%s` instance.\n%s'
% (cls._class_name, errors))
raise InvalidDocumentError(msg)
# In STRICT documents, remove any keys that aren't in cls._fields
if cls.STRICT:
data = {k: v for k, v in data.items() if k in cls._fields}
obj = cls(__auto_convert=False, _created=created, __only_fields=only_fields, **data)
obj._changed_fields = changed_fields
if not _auto_dereference:
obj._fields = fields
return obj
def ajax_reference(self, class_name, id):
oid = ObjectId(id)
doc = get_document(class_name)
obj = doc.objects.get(id=oid)
return Response(json.dumps({
'label': str(obj),
'id': str(obj.id),
'collection': obj.to_dbref().collection,
'url': url_for(obj.view_endpoint + ".edit_view", id=obj.id)
}), content_type='application/json')
def cls_by_son(cls, son):
"""Return class for the bson.son.SON instance provided. Use `_cls` as a class name by default. Override to customize the instance creation (known children handling)"""
try:
class_name = son['_cls']
except (TypeError, KeyError, ValueError):
class_name = cls._class_name
# Return correct subclass for document type
if class_name != cls._class_name:
cls = get_document(class_name)
return cls
def cls_by_son(cls, son):
"""Return class for the bson.son.SON instance provided. Use `_cls` as a class name by default. Override to customize the instance creation (known children handling)"""
try:
class_name = son['_cls']
except (TypeError, KeyError, ValueError):
class_name = cls._class_name
# Return correct subclass for document type
if class_name != cls._class_name:
cls = get_document(class_name)
return cls
def backup_json(collname):
currDate = datetime.now().strftime("%y%m%d-%H:%M")
filename = collname + "_" + currDate + ".json"
model = get_document(collname)
with open("data/backups/" + filename, 'w') as f:
f.write(model.objects.to_json() + '\n')
logging.info('BACKUP : Backup to ' + filename)
return 0
def _resolve_model(obj):
"""
Inherited from rest_framework.utils.model_meta
Overridden for MongoDB compability
"""
if isinstance(obj, six.string_types) and len(obj.split('.')) == 2:
app_name, model_name = obj.split('.')
resolved_model = get_document(model_name)
if resolved_model is None:
msg = "Mongoengine did not return a model for {0}.{1}"
raise ImproperlyConfigured(msg.format(app_name, model_name))
return resolved_model
elif inspect.isclass(obj) and issubclass(obj, mongoengine.BaseDocument):
return obj
raise ValueError("{0} is not a MongoDB Document".format(obj))
def _resolve_model(obj):
"""
Inherited from rest_framework.utils.model_meta
Overridden for MongoDB compability
"""
if isinstance(obj, six.string_types) and len(obj.split('.')) == 2:
app_name, model_name = obj.split('.')
resolved_model = get_document(model_name)
if resolved_model is None:
msg = "Mongoengine did not return a model for {0}.{1}"
raise ImproperlyConfigured(msg.format(app_name, model_name))
return resolved_model
elif inspect.isclass(obj) and issubclass(obj, mongoengine.BaseDocument):
return obj
raise ValueError("{0} is not a MongoDB Document".format(obj))
def to_internal_value(self, data):
"""
Dict of native values <- Dict of primitive datatypes.
"""
if not isinstance(data, dict):
message = self.error_messages['invalid'].format(
datatype=type(data).__name__)
raise drf_fields.ValidationError(
{api_settings.NON_FIELD_ERRORS_KEY: [message]})
ret = OrderedDict()
errors = OrderedDict()
if data['_cls']:
cls = get_document(data['_cls'])
if not issubclass(cls, self.Meta.model):
#not playing the 'pass anything in, and we'll construct it for you' game
cls = self.Meta.model
else:
cls = self.Meta.model
fields = self.chainmap[cls]
fields = [
field for name, field in fields.items() if not field.read_only
]
for field in fields:
validate_method = getattr(self, 'validate_' + field.field_name,
None)
primitive_value = field.get_value(data)
try:
validated_value = field.run_validation(primitive_value)
if validate_method is not None:
validated_value = validate_method(validated_value)
except drf_fields.ValidationError as exc:
errors[field.field_name] = exc.detail
except drf_fields.DjangoValidationError as exc:
errors[field.field_name] = list(exc.messages)
except SkipField:
pass
else:
drf_fields.set_value(ret, field.source_attrs, validated_value)
if errors:
raise drf_fields.ValidationError(errors)
return ret
def to_internal_value(self, data):
"""
Dict of native values <- Dict of primitive datatypes.
"""
if not isinstance(data, dict):
message = self.error_messages['invalid'].format(
datatype=type(data).__name__)
raise drf_fields.ValidationError(
{api_settings.NON_FIELD_ERRORS_KEY: [message]})
if data.get('_cls', None):
cls = get_document(data['_cls'])
else:
cls = self.Meta.model
return self.get_serializer(cls).to_internal_value(data)
def to_internal_value(self, data):
"""
Dict of native values <- Dict of primitive datatypes.
"""
if not isinstance(data, dict):
message = self.error_messages['invalid'].format(
datatype=type(data).__name__
)
raise drf_fields.ValidationError({
api_settings.NON_FIELD_ERRORS_KEY: [message]
})
ret = OrderedDict()
errors = OrderedDict()
if data['_cls']:
cls = get_document(data['_cls'])
if not issubclass(cls, self.Meta.model):
#not playing the 'pass anything in, and we'll construct it for you' game
cls = self.Meta.model
else:
cls = self.Meta.model
fields = self.chainmap[cls]
fields = [field for name, field in fields.items() if not field.read_only]
for field in fields:
validate_method = getattr(self, 'validate_' + field.field_name, None)
primitive_value = field.get_value(data)
try:
validated_value = field.run_validation(primitive_value)
if validate_method is not None:
validated_value = validate_method(validated_value)
except drf_fields.ValidationError as exc:
errors[field.field_name] = exc.detail
except drf_fields.DjangoValidationError as exc:
errors[field.field_name] = list(exc.messages)
except SkipField:
pass
else:
drf_fields.set_value(ret, field.source_attrs, validated_value)
if errors:
raise drf_fields.ValidationError(errors)
return ret
def _from_son(cls, son, _auto_dereference=True):
"""Create an instance of a Document (subclass) from a PyMongo SON.
"""
# get the class name from the document, falling back to the given
# class if unavailable
class_name = son.get('_cls', cls._class_name)
data = dict(("%s" % key, value) for key, value in son.iteritems())
if not UNICODE_KWARGS:
# python 2.6.4 and lower cannot handle unicode keys
# passed to class constructor example: cls(**data)
to_str_keys_recursive(data)
# Return correct subclass for document type
if class_name != cls._class_name:
cls = get_document(class_name)
#Apply the traits
traits = son.get('_traits', [])
if traits:
cls = apply_traits(cls, traits)
changed_fields = []
errors_dict = {}
fields = cls._fields
if not _auto_dereference:
fields = copy.copy(fields)
for field_name, field in fields.iteritems():
field._auto_dereference = _auto_dereference
if field.db_field in data:
value = data[field.db_field]
try:
data[field_name] = (value if value is None
else field.to_python(value))
if field_name != field.db_field:
del data[field.db_field]
except (AttributeError, ValueError), e:
errors_dict[field_name] = e
elif field.default:
default = field.default
if callable(default):
default = default()
if isinstance(default, BaseDocument):
changed_fields.append(field_name)
def __expand_dynamic_values(self, name, value):
"""expand any dynamic values to their correct types / values"""
if not isinstance(value, (dict, list, tuple)):
return value
EmbeddedDocumentListField = _import_class('EmbeddedDocumentListField')
is_list = False
if not hasattr(value, 'items'):
is_list = True
value = dict([(k, v) for k, v in enumerate(value)])
if not is_list and '_cls' in value:
doc_cls = get_document(value['_cls'])
if '_ref' in value and isinstance(value['_ref'], DBRef):
reference = value['_ref']
doc = doc_cls._get_db().dereference(reference)
if doc is not None:
doc = doc_cls._from_son(doc)
return doc
else:
return doc_cls(**value)
data = {}
for k, v in value.items():
key = name if is_list else k
data[k] = self.__expand_dynamic_values(key, v)
if is_list: # Convert back to a list
data_items = sorted(data.items(), key=operator.itemgetter(0))
value = [v for k, v in data_items]
else:
value = data
# Convert lists / values so we can watch for any changes on them
if (isinstance(value, (list, tuple)) and
not isinstance(value, BaseList)):
if issubclass(type(self), EmbeddedDocumentListField):
value = EmbeddedDocumentList(value, self, name)
else:
value = BaseList(value, self, name)
elif isinstance(value, dict) and not isinstance(value, BaseDict):
value = BaseDict(value, self, name)
return value
def get_labels(collname):
model = get_document(collname)
all_labels = model.objects(label__exists = True).only("label")
label_set = []
for lbl in all_labels:
if(lbl["label"] not in label_set):
label_set.append(lbl["label"])
num_of_cl = []
for labl in label_set:
num_of_cl.append(model.objects(label = labl).count())
labellist = zip(label_set, num_of_cl)
return labellist
def to_internal_value(self, data):
"""
Dict of native values <- Dict of primitive datatypes.
"""
if not isinstance(data, dict):
message = self.error_messages['invalid'].format(
datatype=type(data).__name__
)
raise drf_fields.ValidationError({
api_settings.NON_FIELD_ERRORS_KEY: [message]
})
if data.get('_cls', None):
cls = get_document(data['_cls'])
else:
cls = self.Meta.model
return self.get_serializer(cls).to_internal_value(data)
def __init__(self, native_data=None, data=None, many=False):
"""
:param object native_data: Object or list of objects to serialize
:param dict data: Optional serialized data
:param bool many: Define if native_data is an object or a list of objects
:raises AttributeError: If model is not defined in a subclassed instance
"""
if self.model is None:
raise AttributeError('model must be defined in serializer class')
else:
self.model = get_document(self.model)
if not hasattr(self, 'fields') or self.fields is None:
self.fields = self.model._fields
self.data = data
self.native_data = native_data
self.many = many
if self.native_data is not None:
self.data = self.serialize()
def get_step_data(collname, num, page=None):
model = get_document(collname)
tweets = []
if(page == "Cluster_Them"):
for item in model.objects[5:num+5]:
for i in item["ctweettuplelist"][:10]:
tweets.append(i[2])
tweets.append("------------------")
else:
for item in model.objects[5:num+5]:
tweets.append(item["text"])
length = model.objects.count()
return tweets, length
def _from_son(cls, son, _auto_dereference=True):
"""Create an instance of a Document (subclass) from a PyMongo SON.
"""
# get the class name from the document, falling back to the given
# class if unavailable
class_name = son.get('_cls', cls._class_name)
data = dict(("%s" % key, value) for key, value in son.iteritems())
if not UNICODE_KWARGS:
# python 2.6.4 and lower cannot handle unicode keys
# passed to class constructor example: cls(**data)
to_str_keys_recursive(data)
# Return correct subclass for document type
if class_name != cls._class_name:
cls = get_document(class_name)
changed_fields = []
errors_dict = {}
fields = cls._fields
if not _auto_dereference:
fields = copy.copy(fields)
for field_name, field in fields.iteritems():
field._auto_dereference = _auto_dereference
if field.db_field in data:
value = data[field.db_field]
try:
data[field_name] = (value if value is None else
field.to_python(value))
if field_name != field.db_field:
del data[field.db_field]
except (AttributeError, ValueError), e:
errors_dict[field_name] = e
elif field.default:
default = field.default
if callable(default):
default = default()
if isinstance(default, BaseDocument):
changed_fields.append(field_name)
def populate_obj(self, obj, name):
class_ = get_document(self.data['class_name'])
candidate = class_(pk=ObjectId(self.data['id']))
setattr(obj, name, candidate)
def _from_son(cls,
son,
_auto_dereference=True,
only_fields=None,
created=False):
"""Create an instance of a Document (subclass) from a PyMongo SON."""
if not only_fields:
only_fields = []
if son and not isinstance(son, dict):
raise ValueError(
"The source SON object needs to be of type 'dict' but a '%s' was found"
% type(son))
# Get the class name from the document, falling back to the given
# class if unavailable
class_name = son.get("_cls", cls._class_name)
# Convert SON to a data dict, making sure each key is a string and
# corresponds to the right db field.
data = {}
for key, value in son.items():
key = str(key)
key = cls._db_field_map.get(key, key)
data[key] = value
# Return correct subclass for document type
if class_name != cls._class_name:
cls = get_document(class_name)
errors_dict = {}
fields = cls._fields
if not _auto_dereference:
fields = copy.deepcopy(fields)
for field_name, field in fields.items():
field._auto_dereference = _auto_dereference
if field.db_field in data:
value = data[field.db_field]
try:
data[field_name] = (value if value is None else
field.to_python(value))
if field_name != field.db_field:
del data[field.db_field]
except (AttributeError, ValueError) as e:
errors_dict[field_name] = e
if errors_dict:
errors = "\n".join([
"Field '{}' - {}".format(k, v) for k, v in errors_dict.items()
])
msg = "Invalid data to create a `{}` instance.\n{}".format(
cls._class_name,
errors,
)
raise InvalidDocumentError(msg)
# In STRICT documents, remove any keys that aren't in cls._fields
if cls.STRICT:
data = {k: v for k, v in data.items() if k in cls._fields}
obj = cls(__auto_convert=False,
_created=created,
__only_fields=only_fields,
**data)
obj._changed_fields = []
if not _auto_dereference:
obj._fields = fields
return obj
def get_randomcluster(collname, is_labeled): # !! reorganize here.. DRY!! random_cluster = None current_label = "" warning = "" top10 = [] last10 = [] model = get_document(collname) if(is_labeled == "True"): num_of_clusters = model.objects(label__exists = True).count() if(num_of_clusters > 0): rand = random.randint(0, num_of_clusters-1) random_cluster = model.objects(label__exists = True)[rand] current_label = random_cluster["label"] tweetlist = [] for cl in random_cluster["ctweettuplelist"]: tweetlist.append(cl[2]) if(len(tweetlist) > 20): top10 = tweetlist[:10] last10 = tweetlist[-10:] else: top10 = tweetlist #All tweets else: warning = "There is not any labeled cluster yet" elif(is_labeled == "False"): num_of_clusters = model.objects(label__exists = False).count() if(num_of_clusters > 0): rand = random.randint(0, num_of_clusters-1) random_cluster = model.objects(label__exists = False)[rand] tweetlist = [] for cl in random_cluster["ctweettuplelist"]: tweetlist.append(cl[2]) if(len(tweetlist) > 20): top10 = tweetlist[:10] last10 = tweetlist[-10:] else: top10 = tweetlist #All tweets else: warning = "All clusters are labeled." return random_cluster, top10, last10, current_label, warning
def post(self, request, collname, is_labeled):
if "confirmpass" in request.POST:
user_pass = request.POST['user_pass']
reset_pass = config.get('dataset', 'reset_pass')
if(user_pass == reset_pass):
random_cluster, top10, last10, current_label, warning = get_randomcluster(collname, is_labeled)
labellist = get_labels(collname)
return render(request, 'label.html', {
'random_cluster' : random_cluster,
'top10' : top10,
'last10' : last10,
'labellist' : labellist,
'collname' : collname,
'is_labeled': is_labeled,
'current_label' : current_label,
'warning' : warning,
})
else:
confirmed = False
action = "labeling"
event = 'see the clusters for '
denied_msg = "Wrong password. Please try again."
return render(request, 'confirmpass.html', {
'action': action,
'event' : event,
'thing' : collname,
'confirmed' : confirmed,
'denied_msg' : denied_msg,
'is_labeled' : is_labeled,
})
elif "label" in request.POST:
client_address = get_client_ip(request)
#Add the label to DB
input_label = request.POST['label']
cl_id = request.POST['cl_id']
model = get_document(collname)
if(input_label==""):
model.objects.get(pk=cl_id).update(unset__label = 1)
logging.info('LABEL: Label deleted from ' + cl_id + ', by ' + client_address + ' for ' + collname)
else:
model.objects.get(pk=cl_id).update(set__label = str(input_label))
logging.info('LABEL: ' + cl_id + ' labeled as "' + input_label + '", by ' + client_address + ' for ' + collname)
random_cluster, top10, last10, current_label, warning = get_randomcluster(collname, is_labeled)
labellist = get_labels(collname)
return render(request, 'label.html', {
'random_cluster' : random_cluster,
'top10' : top10,
'last10' : last10,
'labellist' : labellist,
'collname' : collname,
'is_labeled': is_labeled,
'current_label' : current_label,
'warning' : warning,
})
elif "directlabel" in request.POST:
client_address = get_client_ip(request)
#Add the label to DB
input_label = request.POST['directlabel']
cl_id = request.POST['cl_id']
model = get_document(collname)
model.objects.get(pk=cl_id).update(set__label = str(input_label))
logging.info('LABEL: ' + cl_id + ' labeled as "' + input_label + '", by ' + client_address + ' for ' + collname)
random_cluster, top10, last10, current_label, warning = get_randomcluster(collname, is_labeled)
labellist = get_labels(collname)
return render(request, 'label.html', {
'random_cluster' : random_cluster,
'top10' : top10,
'last10' : last10,
'labellist' : labellist,
'collname' : collname,
'is_labeled': is_labeled,
'current_label' : current_label,
'warning' : warning,
})
elif "nextcl" in request.POST:
random_cluster, top10, last10, current_label, warning = get_randomcluster(collname, is_labeled)
labellist = get_labels(collname)
return render(request, 'label.html', {
'random_cluster' : random_cluster,
'top10' : top10,
'last10' : last10,
'labellist' : labellist,
'collname' : collname,
'is_labeled': is_labeled,
'current_label' : current_label,
'warning' : warning,
})
