def handle(self, *args, **kwargs):
xforms = XForm.objects.filter(instances_with_osm=True)
# username
if args:
xforms = xforms.filter(user__username=args[0])
for xform in queryset_iterator(xforms):
attachments = Attachment.objects.filter(
extension=Attachment.OSM,
instance__xform=xform).distinct('instance')
count = attachments.count()
c = 0
for a in queryset_iterator(attachments):
pk = a.instance.parsed_instance.pk
save_osm_data(pk)
c += 1
if c % 1000 == 0:
self.stdout.write(
"%s:%s: Processed %s of %s." %
(xform.user.username, xform.id_string, c, count))
self.stdout.write("%s:%s: processed %s of %s submissions." %
(xform.user.username, xform.id_string, c, count))
def handle(self, *args, **options):
deleted = 0
self.stdout.write("Starting UserObject")
for perm in queryset_iterator(
UserObjectPermission.objects.select_related()):
try:
perm.content_object
except AttributeError:
perm.delete()
deleted += 1
self.stdout.write(
"deleted {} stale permission".format(deleted)
)
self.stdout.write("Starting GroupObject")
for perm in queryset_iterator(
GroupObjectPermission.objects.select_related()):
try:
perm.content_object
except AttributeError:
perm.delete()
deleted += 1
self.stdout.write(
"deleted {} stale permission".format(deleted)
)
self.stdout.write(
"Total removed orphan object permissions instances: %d" % deleted
)
def handle(self, *args, **options):
self.stdout.write("Re-assigining started", ending='\n')
if not args:
raise CommandError('Param not set. <app model [created_perm]>')
if len(args) < 3:
raise CommandError('Param not set. <app model [created_perm]>')
app = args[0]
model = args[1]
username = args[2]
new_perms = list(args[3:])
if username == "all":
users = User.objects.exclude(
username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME
)
teams = Team.objects.all()
else:
users = User.objects.filter(username=username)
teams = Team.objects.filter(organization__username=username)
# Get all the users
for user in queryset_iterator(users):
self.reassign_perms(user, app, model, new_perms)
for team in queryset_iterator(teams):
self.reassign_perms(team, app, model, new_perms)
self.stdout.write("Re-assigining finished", ending='\n')
def handle(self, *args, **options):
# XForms
for xform in queryset_iterator(XForm.objects.all()):
OwnerRole.add(xform.user, xform)
# UserProfile
for profile in queryset_iterator(UserProfile.objects.all()):
set_api_permissions_for_user(profile.user)
OwnerRole.add(profile.user, profile)
if profile.created_by is not None:
OwnerRole.add(profile.created_by, profile)
# OrganizationProfile
for profile in queryset_iterator(OrganizationProfile.objects.all()):
OwnerRole.add(profile.user, profile)
if profile.created_by is not None:
OwnerRole.add(profile.created_by, profile)
if profile.creator is not None:
OwnerRole.add(profile.creator, profile)
# Project
for project in queryset_iterator(Project.objects.all()):
OwnerRole.add(project.organization, project)
OwnerRole.add(project.created_by, project)
def handle(self, *args, **kwargs):
xforms = XForm.objects.filter(instances_with_osm=True)
# username
if args:
xforms = xforms.filter(user__username=args[0])
for xform in queryset_iterator(xforms):
attachments = Attachment.objects.filter(
extension=Attachment.OSM,
instance__xform=xform
).distinct('instance')
count = attachments.count()
c = 0
for a in queryset_iterator(attachments):
pk = a.instance.parsed_instance.pk
save_osm_data(pk)
c += 1
if c % 1000 == 0:
self.stdout.write("%s:%s: Processed %s of %s." %
(xform.user.username,
xform.id_string, c, count))
self.stdout.write("%s:%s: processed %s of %s submissions." %
(xform.user.username, xform.id_string, c, count))
def handle(self, *args, **options):
print "Task started ..."
# Get all the users
for user in queryset_iterator(User.objects.all()):
# For each user get the forms which are projectless
for form in queryset_iterator(
XForm.objects.select_related('projectxform').filter(
projectxform=None, user=user)):
# Create the default project
self.create_and_assign_project(user, form)
print "Task completed ..."
def handle(self, *args, **options):
print "Task started ..."
# Get all the users
for user in queryset_iterator(User.objects.all()):
# For each user get the forms which are projectless
for form in queryset_iterator(XForm.objects
.select_related('projectxform')
.filter(projectxform=None,
user=user)):
# Create the default project
self.create_and_assign_project(user, form)
print "Task completed ..."
def destroy(self, request, *args, **kwargs):
instance_ids = request.data.get('instance_ids')
self.object = self.get_object()
if isinstance(self.object, XForm):
if not instance_ids:
raise ParseError(_(u"Data id(s) not provided."))
else:
instance_ids = [
x for x in instance_ids.split(',') if x.isdigit()
]
if not instance_ids:
raise ParseError(_(u"Invalid data ids were provided."))
initial_count = self.object.submission_count()
queryset = Instance.objects.filter(
id__in=instance_ids,
xform=self.object,
# do not update this timestamp when the record have
# already been deleted.
deleted_at__isnull=True)
# loop through queryset
# then call delete_instance that calls .save()
# to allow emitting post_save signal
for instance in queryset_iterator(queryset):
delete_instance(instance, request.user)
# updates the num_of_submissions for the form.
after_count = self.object.submission_count(force_update=True)
number_of_records_deleted = initial_count - after_count
# send message
send_message(instance_id=instance_ids,
target_id=self.object.id,
target_type=XFORM,
user=request.user,
message_verb=SUBMISSION_DELETED)
return Response(data={
"message":
"%d records were deleted" % number_of_records_deleted
},
status=status.HTTP_200_OK)
elif isinstance(self.object, Instance):
if request.user.has_perm(CAN_DELETE_SUBMISSION, self.object.xform):
delete_instance(self.object, request.user)
# send message
send_message(instance_id=self.object,
target_id=self.object.xform.id,
target_type=XFORM,
user=request.user,
message_verb=SUBMISSION_DELETED)
else:
raise PermissionDenied(
_(u"You do not have delete "
u"permissions."))
return Response(status=status.HTTP_204_NO_CONTENT)
def _get_streaming_response(self):
"""
Get a StreamingHttpResponse response object
"""
# use queryset_iterator. Will need to change this to the Django
# native .iterator() method when we upgrade to django version 2
# because in Django 2 .iterator() has support for chunk size
queryset = queryset_iterator(self.object_list, chunksize=2000)
def get_json_string(item):
return json.dumps(XFormBaseSerializer(
instance=item,
context={'request': self.request}
).data)
response = StreamingHttpResponse(
json_stream(queryset, get_json_string),
content_type="application/json"
)
# calculate etag value and add it to response headers
if hasattr(self, 'etag_data'):
self.set_etag_header(None, self.etag_data)
self.set_cache_control(response)
# set headers on streaming response
for k, v in self.headers.items():
response[k] = v
return response
def handle(self, *args, **options):
self.stdout.write("Assign permission to the member team", ending='\n')
count = 0
fail = 0
total = 0
if args:
try:
org_name = args[0]
org = OrganizationProfile.objects.get(user__username=org_name)
team = Team.objects.get(organization=org.user,
name=u'%s#%s' % (
org.user.username,
'members'))
self.assign_perm(team, org)
count += 1
total += 1
except ObjectDoesNotExist as e:
fail += 1
self.stdout.write(str(e), ending='\n')
else:
# Get all the teams
for team in queryset_iterator(
Team.objects.filter(name__contains='members')):
self.assign_perm(team, team.organization)
count += 1
total += 1
self.stdout.write("Assigned {} of {} records. failed: {}".
format(count, total, fail), ending='\n')
def handle(self, *args, **options):
self.stdout.write("Assign permission to the member team", ending='\n')
count = 0
fail = 0
total = 0
if args:
try:
org_name = args[0]
org = OrganizationProfile.objects.get(user__username=org_name)
team = Team.objects.get(organization=org.user,
name=u'%s#%s' %
(org.user.username, 'members'))
self.assign_perm(team, org)
count += 1
total += 1
except ObjectDoesNotExist as e:
fail += 1
self.stdout.write(str(e), ending='\n')
else:
# Get all the teams
for team in queryset_iterator(
Team.objects.filter(name__contains='members')):
self.assign_perm(team, team.organization)
count += 1
total += 1
self.stdout.write("Assigned {} of {} records. failed: {}".format(
count, total, fail),
ending='\n')
def _get_streaming_response(self):
"""
Get a StreamingHttpResponse response object
"""
# use queryset_iterator. Will need to change this to the Django
# native .iterator() method when we upgrade to django version 2
# because in Django 2 .iterator() has support for chunk size
queryset = queryset_iterator(self.object_list, chunksize=2000)
def get_json_string(item):
return json.dumps(
XFormBaseSerializer(instance=item,
context={
'request': self.request
}).data)
response = StreamingHttpResponse(json_stream(queryset,
get_json_string),
content_type="application/json")
# calculate etag value and add it to response headers
if hasattr(self, 'etag_data'):
self.set_etag_header(None, self.etag_data)
self.set_cache_control(response)
# set headers on streaming response
for k, v in self.headers.items():
response[k] = v
return response
def handle(self, *args, **options):
print "Task started ..."
# Get all the users
for user in queryset_iterator(
User.objects.exclude(pk=settings.ANONYMOUS_USER_ID)):
self.set_project_to_user_forms(user)
print "Task completed ..."
def handle(self, *args, **options):
print "Task started ..."
# Get all the users
for user in queryset_iterator(
User.objects.exclude(pk=settings.ANONYMOUS_USER_ID)):
self.set_project_to_user_forms(user)
print "Task completed ..."
def process_attachments(self, user):
"""
Process attachments for submissions where media_all_received is False.
"""
xforms = XForm.objects.filter(user=user,
deleted_at__isnull=True,
downloadable=True)
for xform in queryset_iterator(xforms):
submissions = xform.instances.filter(media_all_received=False)
to_process = submissions.count()
if to_process:
for submission in queryset_iterator(submissions):
update_attachments(submission)
not_processed = xform.instances.filter(
media_all_received=False).count()
self.stdout.write("%s to process %s - %s = %s processed" %
(xform, to_process, not_processed,
(to_process - not_processed)))
def forwards(self, orm):
"""Add parsed JSON to JSON instance column."""
for instance in queryset_iterator(orm["odk_logger.Instance"].objects.all()):
obj = Instance.objects.get(pk=instance.pk)
json = obj.get_dict()
json[SUBMISSION_TIME] = instance.date_created.strftime(MONGO_STRFTIME)
json[XFORM_ID_STRING] = obj._parser.get_xform_id_string()
instance.json = json
instance.save()
def handle(self, *args, **kwargs):
self.stdout.write(_('%(nb)d XForms to update')
% {'nb': DataDictionary.objects.count()})
for i, dd in enumerate(
queryset_iterator(DataDictionary.objects.all())):
if dd.xls:
dd._set_uuid_in_xml()
super(DataDictionary, dd).save()
if (i + 1) % 10 == 0:
self.stdout.write(_('Updated %(nb)d XForms...') % {'nb': i})
def handle(self, *args, **options):
print "Task started ..."
users = User.objects.exclude(
username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME)
# Get all the users
for user in queryset_iterator(users):
self.set_project_to_user_forms(user)
print "Task completed ..."
def handle(self, *args, **kwargs):
print('%(nb)d XForms to update' %
{'nb': DataDictionary.objects.count()})
for i, dd in enumerate(queryset_iterator(
DataDictionary.objects.all())):
if dd.xls:
dd.set_uuid_in_xml(id_string=dd.id_string)
super(DataDictionary, dd).save()
if (i + 1) % 10 == 0:
print('Updated %(nb)d XForms...' % {'nb': i})
def handle(self, *args, **options):
self.stdout.write("Migrate permissions started", ending='\n')
if len(args) < 2:
self.stdout.write("This command takes two argument -m and -p "
"Example: "
"-m logger.Team "
"-p logger.TeamUserObjectPermission")
exit()
if options['app_model']:
app_model = args[0]
else:
self.stdout.write("-m , should be set as the first argument")
exit()
if options['perms_tbl']:
perms_tbl = args[1]
else:
self.stdout.write("-p , should be set as the second argument")
exit()
model = get_model(app_model)
perms_model = get_model(perms_tbl)
if not issubclass(perms_model, UserObjectPermissionBase):
self.stdout.write("-p , should be a model of a class that is "
"a subclass of UserObjectPermissionBase")
exit()
ct = ContentType.objects.get(model=model.__name__.lower(),
app_label=model._meta.app_label)
# Get all the users
users = User.objects.exclude(
username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME
).order_by('username')
for user in queryset_iterator(users):
self.stdout.write(
"Processing: {} - {}".format(user.pk, user.username)
)
for uop in user.userobjectpermission_set.filter(content_type=ct)\
.select_related('permission', 'content_type')\
.prefetch_related('permission', 'content_type'):
try:
perms_model(
content_object=uop.content_object,
user=user,
permission=uop.permission
).save()
except IntegrityError:
continue
except ValueError:
pass
def handle(self, *args, **options):
print "Task started ..."
# Delete old bamboo rest service
RestService.objects.filter(name='bamboo').delete()
# Get all the rest services
for rest in queryset_iterator(RestService.objects.all()):
rest.save()
print "Task ended ..."
def handle(self, *args, **options):
self.stdout.write("Task started ...")
users = User.objects.exclude(
username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME
)
# Get all the users
for user in queryset_iterator(users):
self.set_project_to_user_forms(user)
self.stdout.write("Task completed ...")
def handle(self, *args, **kwargs):
self.stdout.write(
'"username","email","first_name","last_name","name","organization"'
)
for p in queryset_iterator(UserProfile.objects.all()):
self.stdout.write(
u'"{}","{}","{}","{}","{}","{}"'.format(
p.user.username, p.user.email, p.user.first_name,
p.user.last_name, p.name, p.organization
)
)
def forwards(self, orm):
"""Add parsed JSON to JSON instance column."""
for instance in queryset_iterator(
orm['odk_logger.Instance'].objects.all()):
obj = Instance.objects.get(pk=instance.pk)
json = obj.get_dict()
json[SUBMISSION_TIME] = instance.date_created.strftime(
MONGO_STRFTIME)
json[XFORM_ID_STRING] = obj._parser.get_xform_id_string()
instance.json = json
instance.save()
def handle(self, *args, **options):
self.stdout.write("Task started ...")
# Delete old bamboo rest service
RestService.objects.filter(name='bamboo').delete()
# Get all the rest services
for rest in queryset_iterator(RestService.objects.all()):
rest.save()
self.stdout.write("Task ended ...")
def handle(self, *args, **kwargs):
self.stdout.write(
_('%(nb)d XForms to update') %
{'nb': DataDictionary.objects.count()})
for i, dd in enumerate(queryset_iterator(
DataDictionary.objects.all())):
if dd.xls:
dd._set_uuid_in_xml()
super(DataDictionary, dd).save()
if (i + 1) % 10 == 0:
self.stdout.write(_('Updated %(nb)d XForms...') % {'nb': i})
def handle(self, *args, **options):
attachments_qs = Attachment.objects.select_related(
'instance', 'instance__xform')
if options.get('username'):
username = options.get('username')
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
raise CommandError(
"Error: username %(username)s does not exist" %
{'username': username})
attachments_qs = attachments_qs.filter(instance__user=user)
if options.get('id_string'):
id_string = options.get('id_string')
try:
xform = XForm.objects.get(id_string=id_string)
except XForm.DoesNotExist:
raise CommandError(
"Error: Form with id_string %(id_string)s does not exist" %
{'id_string': id_string})
attachments_qs = attachments_qs.filter(instance__xform=xform)
fs = get_storage_class('django.core.files.storage.FileSystemStorage')()
for att in queryset_iterator(attachments_qs):
filename = att.media_file.name
default_storage = get_storage_class()()
full_path = get_path(filename,
settings.THUMB_CONF['small']['suffix'])
if options.get('force') is not None:
for s in ['small', 'medium', 'large']:
fp = get_path(filename, settings.THUMB_CONF[s]['suffix'])
if default_storage.exists(fp):
default_storage.delete(fp)
if not default_storage.exists(full_path):
try:
if default_storage.__class__ != fs.__class__:
resize(filename, att.extension)
else:
resize_local_env(filename, att.extension)
path = get_path(filename,
'%s' % THUMB_CONF['small']['suffix'])
if default_storage.exists(path):
self.stdout.write(
_(u'Thumbnails created for %(file)s') %
{'file': filename})
else:
self.stdout.write(
_(u'Problem with the file %(file)s') %
{'file': filename})
except (IOError, OSError) as e:
self.stderr.write(
_(u'Error on %(filename)s: %(error)s') % {
'filename': filename,
'error': e
})
def handle(self, *args, **options):
self.stdout.write("Migrate permissions started", ending='\n')
if len(args) < 2:
self.stdout.write("This command takes two argument -m and -p "
"Example: "
"-m logger.Team "
"-p logger.TeamUserObjectPermission")
exit()
if options['app_model']:
app_model = args[0]
else:
self.stdout.write("-m , should be set as the first argument")
exit()
if options['perms_tbl']:
perms_tbl = args[1]
else:
self.stdout.write("-p , should be set as the second argument")
exit()
model = get_model(app_model)
perms_model = get_model(perms_tbl)
if not issubclass(perms_model, UserObjectPermissionBase):
self.stdout.write("-p , should be a model of a class that is "
"a subclass of UserObjectPermissionBase")
exit()
ct = ContentType.objects.get(
model=model.__name__.lower(), app_label=model._meta.app_label)
# Get all the users
users = User.objects.exclude(
username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME).order_by(
'username')
for user in queryset_iterator(users):
self.stdout.write(
"Processing: {} - {}".format(user.pk, user.username))
for uop in user.userobjectpermission_set.filter(content_type=ct)\
.select_related('permission', 'content_type')\
.prefetch_related('permission', 'content_type'):
try:
perms_model(
content_object=uop.content_object,
user=user,
permission=uop.permission).save()
except IntegrityError:
continue
except ValueError:
pass
def handle(self, *args, **kwargs):
attachments_qs = Attachment.objects.select_related(
'instance', 'instance__xform')
if kwargs.get('username'):
username = kwargs.get('username')
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
raise CommandError(
"Error: username %(username)s does not exist" %
{'username': username})
attachments_qs = attachments_qs.filter(instance__user=user)
if kwargs.get('id_string'):
id_string = kwargs.get('id_string')
try:
xform = XForm.objects.get(id_string=id_string)
except XForm.DoesNotExist:
raise CommandError(
"Error: Form with id_string %(id_string)s does not exist" %
{'id_string': id_string})
attachments_qs = attachments_qs.filter(instance__xform=xform)
for att in queryset_iterator(attachments_qs):
filename = att.media_file.name
default_storage = get_storage_class()()
full_path = get_path(filename,
settings.THUMB_CONF['small']['suffix'])
if kwargs.get('force') is not None:
for s in settings.THUMB_CONF.keys():
fp = get_path(filename, settings.THUMB_CONF[s]['suffix'])
if default_storage.exists(fp):
default_storage.delete(fp)
if not default_storage.exists(full_path):
try:
resize(filename)
if default_storage.exists(
get_path(
filename, '%s' %
settings.THUMB_CONF['small']['suffix'])):
print(
_('Thumbnails created for %(file)s') %
{'file': filename})
else:
print(
_('Problem with the file %(file)s') %
{'file': filename})
except (IOError, OSError), e:
print(
_('Error on %(filename)s: %(error)s') % {
'filename': filename,
'error': e
})
def kml_export_data(id_string, user, xform=None):
"""
KML export data from form submissions.
"""
def cached_get_labels(xpath):
"""
Get and Cache labels for the XForm.
"""
if xpath in list(labels):
return labels[xpath]
labels[xpath] = xform.get_label(xpath)
return labels[xpath]
xform = xform or XForm.objects.get(id_string=id_string, user=user)
data_kwargs = {'geom__isnull': False}
if xform.is_merged_dataset:
data_kwargs.update({
'xform_id__in':
[i for i in xform.mergedxform.xforms.filter(
deleted_at__isnull=True).values_list('id', flat=True)]
})
else:
data_kwargs.update({'xform_id': xform.pk})
instances = Instance.objects.filter(**data_kwargs).order_by('id')
data_for_template = []
labels = {}
for instance in queryset_iterator(instances):
# read the survey instances
data_for_display = instance.get_dict()
xpaths = list(data_for_display)
xpaths.sort(key=cmp_to_key(instance.xform.get_xpath_cmp()))
table_rows = [
'<tr><td>%s</td><td>%s</td></tr>' %
(cached_get_labels(xpath), data_for_display[xpath]) for xpath in
xpaths if not xpath.startswith(u"_")]
img_urls = image_urls(instance)
if instance.point:
data_for_template.append({
'name': instance.xform.id_string,
'id': instance.id,
'lat': instance.point.y,
'lng': instance.point.x,
'image_urls': img_urls,
'table': '<table border="1"><a href="#"><img width="210" '
'class="thumbnail" src="%s" alt=""></a>%s'
'</table>' % (img_urls[0] if img_urls else "",
''.join(table_rows))})
return data_for_template
def kml_export_data(id_string, user, xform=None):
"""
KML export data from form submissions.
"""
def cached_get_labels(xpath):
"""
Get and Cache labels for the XForm.
"""
if xpath in list(labels):
return labels[xpath]
labels[xpath] = xform.get_label(xpath)
return labels[xpath]
xform = xform or XForm.objects.get(id_string=id_string, user=user)
data_kwargs = {'geom__isnull': False}
if xform.is_merged_dataset:
data_kwargs.update({
'xform_id__in':
[i for i in xform.mergedxform.xforms.filter(
deleted_at__isnull=True).values_list('id', flat=True)]
})
else:
data_kwargs.update({'xform_id': xform.pk})
instances = Instance.objects.filter(**data_kwargs).order_by('id')
data_for_template = []
labels = {}
for instance in queryset_iterator(instances):
# read the survey instances
data_for_display = instance.get_dict()
xpaths = list(data_for_display)
xpaths.sort(key=cmp_to_key(instance.xform.get_xpath_cmp()))
table_rows = [
'<tr><td>%s</td><td>%s</td></tr>' %
(cached_get_labels(xpath), data_for_display[xpath]) for xpath in
xpaths if not xpath.startswith(u"_")]
img_urls = image_urls(instance)
if instance.point:
data_for_template.append({
'name': instance.xform.id_string,
'id': instance.id,
'lat': instance.point.y,
'lng': instance.point.x,
'image_urls': img_urls,
'table': '<table border="1"><a href="#"><img width="210" '
'class="thumbnail" src="%s" alt=""></a>%s'
'</table>' % (img_urls[0] if img_urls else "",
''.join(table_rows))})
return data_for_template
def handle(self, *args, **options):
attachments_qs = Attachment.objects.select_related(
'instance', 'instance__xform')
if options.get('username'):
username = options.get('username')
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
raise CommandError(
"Error: username %(username)s does not exist" %
{'username': username})
attachments_qs = attachments_qs.filter(instance__user=user)
if options.get('id_string'):
id_string = options.get('id_string')
try:
xform = XForm.objects.get(id_string=id_string)
except XForm.DoesNotExist:
raise CommandError(
"Error: Form with id_string %(id_string)s does not exist" %
{'id_string': id_string})
attachments_qs = attachments_qs.filter(instance__xform=xform)
fs = get_storage_class('django.core.files.storage.FileSystemStorage')()
for att in queryset_iterator(attachments_qs):
filename = att.media_file.name
default_storage = get_storage_class()()
full_path = get_path(filename,
settings.THUMB_CONF['small']['suffix'])
if options.get('force') is not None:
for s in ['small', 'medium', 'large']:
fp = get_path(filename, settings.THUMB_CONF[s]['suffix'])
if default_storage.exists(fp):
default_storage.delete(fp)
if not default_storage.exists(full_path):
try:
if default_storage.__class__ != fs.__class__:
resize(filename, att.extension)
else:
resize_local_env(filename, att.extension)
path = get_path(
filename, '%s' % THUMB_CONF['small']['suffix'])
if default_storage.exists(path):
self.stdout.write(
_(u'Thumbnails created for %(file)s') %
{'file': filename})
else:
self.stdout.write(
_(u'Problem with the file %(file)s') %
{'file': filename})
except (IOError, OSError) as e:
self.stderr.write(_(
u'Error on %(filename)s: %(error)s')
% {'filename': filename, 'error': e})
def kml_export_data(id_string, user, xform=None):
if xform is None:
xform = XForm.objects.get(id_string=id_string, user=user)
instances = Instance.objects.filter(xform__user=user,
xform__id_string=id_string,
geom__isnull=False).order_by('id')
data_for_template = []
labels = {}
def cached_get_labels(xpath):
if xpath in labels.keys():
return labels[xpath]
labels[xpath] = xform.get_label(xpath)
return labels[xpath]
for instance in queryset_iterator(instances):
# read the survey instances
data_for_display = instance.get_dict()
xpaths = data_for_display.keys()
xpaths.sort(cmp=instance.xform.get_xpath_cmp())
label_value_pairs = [(cached_get_labels(xpath),
data_for_display[xpath]) for xpath in xpaths
if not xpath.startswith(u"_")]
table_rows = [
'<tr><td>%s</td><td>%s</td></tr>' % (k, v)
for k, v in label_value_pairs
]
img_urls = image_urls(instance)
img_url = img_urls[0] if img_urls else ""
point = instance.point
if point:
data_for_template.append({
'name':
id_string,
'id':
instance.id,
'lat':
point.y,
'lng':
point.x,
'image_urls':
img_urls,
'table':
'<table border="1"><a href="#"><img width="210" '
'class="thumbnail" src="%s" alt=""></a>%s'
'</table>' % (img_url, ''.join(table_rows))
})
return data_for_template
def handle(self, *args, **kwargs):
# username
if args:
users = User.objects.filter(username__contains=args[0])
else:
# All the accounts
self.stdout.write("Fetching all the account {}", ending='\n')
users = queryset_iterator(
User.objects.exclude(pk=settings.ANONYMOUS_USER_ID))
for user in users:
self.add_id(user)
def handle(self, *args, **kwargs):
# username
if args:
users = User.objects.filter(username__contains=args[0])
else:
# All the accounts
self.stdout.write("Fetching all the account {}", ending='\n')
users = queryset_iterator(
User.objects.exclude(pk=settings.ANONYMOUS_USER_ID))
for user in users:
self.add_id(user)
def handle(self, *args, **options):
self.stdout.write("Migrate group permissions started", ending='\n')
if len(args) < 2:
self.stdout.write("This command takes two argument -m and -p "
"Example: "
"-m logger.Team "
"-p logger.TeamUserObjectPermission")
exit()
if options['app_model']:
app_model = args[0]
else:
self.stdout.write("-m , should be set as the first argument")
exit()
if options['perms_tbl']:
perms_tbl = args[1]
else:
self.stdout.write("-p , should be set as the second argument")
exit()
model = get_model(app_model)
perms_model = get_model(perms_tbl)
if not issubclass(perms_model, GroupObjectPermissionBase):
self.stdout.write("-p , should be a model of a class that is "
"a subclass of GroupObjectPermissionBase")
exit()
ct = ContentType.objects.get(
model=model.__name__.lower(), app_label=model._meta.app_label)
teams = Team.objects.filter().annotate(
c=Count('groupobjectpermission')).filter(c__gt=0)
for team in queryset_iterator(teams):
self.stdout.write("Processing: {} - {}".format(team.pk, team.name))
for gop in team.groupobjectpermission_set.filter(content_type=ct)\
.select_related('permission', 'content_type')\
.prefetch_related('permission', 'content_type'):
try:
perms_model(
content_object=gop.content_object,
group=team,
permission=gop.permission).save()
except IntegrityError:
continue
except ValueError:
pass
self.stdout.write("Group permissions migration done.")
def handle(self, *args, **kwargs):
with open('gps_points_export.csv', 'w') as csvfile:
fieldnames = ['longitude', 'latitude', 'date_created']
writer = csv.writer(csvfile)
writer.writerow(fieldnames)
for instance in queryset_iterator(
Instance.objects.exclude(geom__isnull=True)):
if hasattr(instance, 'point') and instance.point is not None:
longitude = instance.point.coords[0]
latitude = instance.point.coords[1]
writer.writerow(
[longitude, latitude, instance.date_created])
self.stdout.write("Export of gps files has completed!!!!")
def handle(self, *args, **kwargs):
xforms = XForm.objects.all()
total = xforms.count()
count = 0
for xform in queryset_iterator(XForm.objects.all()):
has_geo = xform.geocoded_submission_count() > 0
try:
xform.instances_with_geopoints = has_geo
xform.save()
except Exception as e:
self.stderr.write(e)
else:
count += 1
self.stdout.write("%d of %d forms processed." % (count, total))
def handle(self, *args, **options):
self.stdout.write("Migrate group permissions started", ending='\n')
if len(args) < 2:
self.stdout.write("This command takes two argument -m and -p "
"Example: "
"-m logger.Team "
"-p logger.TeamUserObjectPermission")
exit()
if options['app_model']:
app_model = args[0]
else:
self.stdout.write("-m , should be set as the first argument")
exit()
if options['perms_tbl']:
perms_tbl = args[1]
else:
self.stdout.write("-p , should be set as the second argument")
exit()
model = get_model(app_model)
perms_model = get_model(perms_tbl)
if not issubclass(perms_model, GroupObjectPermissionBase):
self.stdout.write("-p , should be a model of a class that is "
"a subclass of GroupObjectPermissionBase")
exit()
ct = ContentType.objects.get(model=model.__name__.lower(),
app_label=model._meta.app_label)
teams = Team.objects.filter().annotate(
c=Count('groupobjectpermission')).filter(c__gt=0)
for team in queryset_iterator(teams):
self.stdout.write("Processing: {} - {}".format(team.pk, team.name))
for gop in team.groupobjectpermission_set.filter(content_type=ct)\
.select_related('permission', 'content_type')\
.prefetch_related('permission', 'content_type'):
try:
perms_model(content_object=gop.content_object,
group=team,
permission=gop.permission).save()
except IntegrityError:
continue
except ValueError:
pass
self.stdout.write("Group permissions migration done.")
def test_queryset_iterator(self):
user_model = get_user_model()
user_model.objects.create_user(username='******',
password='******',
email='*****@*****.**')
user_model.objects.create_user(username='******',
password='******',
email='*****@*****.**')
user_model.objects.create_user(username='******',
password='******',
email='test@test_3.com')
self.assertEquals(
'generator',
queryset_iterator(user_model.objects.all(),
chunksize=1).__class__.__name__)
def handle(self, *args, **options):
deleted = 0
self.stdout.write("Starting UserObject")
for perm in queryset_iterator(
UserObjectPermission.objects.select_related()):
try:
perm.content_object
except AttributeError:
perm.delete()
deleted += 1
self.stdout.write(
"deleted {} stale permission".format(deleted))
self.stdout.write("Starting GroupObject")
for perm in queryset_iterator(
GroupObjectPermission.objects.select_related()):
try:
perm.content_object
except AttributeError:
perm.delete()
deleted += 1
self.stdout.write(
"deleted {} stale permission".format(deleted))
self.stdout.write(
"Total removed orphan object permissions instances: %d" % deleted)
def set_project_to_user_forms(self, user):
default_project_name = user.username + '\'s Project'
try:
project = Project.objects.get(name=default_project_name)
except Project.DoesNotExist:
metadata = {'description': 'Default Project'}
project = Project.objects.create(name=default_project_name,
organization=user,
created_by=user,
metadata=metadata)
self.stdout.write("Created project %s" % project.name)
finally:
xforms = user.xforms.filter(project=XFORM_DEFAULT_PROJECT_ID)
for xform in queryset_iterator(xforms):
xform.project = project
xform.save()
def handle(self, *args, **kwargs):
pks = Attachment.objects.filter(
extension=Attachment.OSM,
instance__xform__instances_with_osm=False)\
.values_list('instance__xform', flat=True).distinct()
xforms = XForm.objects.filter(pk__in=pks)
total = xforms.count()
count = 0
for xform in queryset_iterator(xforms):
try:
xform.instances_with_osm = True
xform.save()
except Exception as e:
print e
else:
count += 1
print "%d of %d forms processed." % (count, total)
def set_project_to_user_forms(self, user):
default_project_name = user.username + '\'s Project'
try:
project = Project.objects.get(name=default_project_name)
except Project.DoesNotExist:
metadata = {'description': 'Default Project'}
project = Project.objects.create(name=default_project_name,
organization=user,
created_by=user,
metadata=metadata)
print "Created project " + project.name
finally:
xforms = user.xforms.filter(project=XFORM_DEFAULT_PROJECT_ID)
for xform in queryset_iterator(xforms):
xform.project = project
xform.save()
def recalculate_xform_hash(apps, schema_editor): # pylint: disable=W0613
"""
Recalculate all XForm hashes.
"""
XForm = apps.get_model('logger', 'XForm') # pylint: disable=C0103
xforms = XForm.objects.filter(downloadable=True,
deleted_at__isnull=True).only('xml')
count = xforms.count()
counter = 0
for xform in queryset_iterator(xforms, 500):
xform.hash = u'md5:%s' % md5(xform.xml.encode('utf8')).hexdigest()
xform.save(update_fields=['hash'])
counter += 1
if counter % 500 == 0:
print("Processed %d of %d forms." % (counter, count))
print("Processed %dforms." % counter)
def handle(self, *args, **kwargs):
self.stdout.write("Updating forms owner", ending='\n')
for project in queryset_iterator(Project.objects.all()):
for xform in project.xform_set.all():
try:
if xform.user != project.organization:
self.stdout.write(
"Processing: {} - {}".format(xform.id_string,
xform.user.username)
)
xform.user = project.organization
xform.save()
except:
self.stdout.write(
"Error processing: {} - {}".format(xform.id_string,
xform.user.username)
)
pass
def handle(self, *args, **kwargs):
self.stdout.write("Updating forms owner", ending='\n')
for project in queryset_iterator(Project.objects.all()):
for xform in project.xform_set.all():
try:
if xform.user != project.organization:
self.stdout.write(
"Processing: {} - {}".format(xform.id_string,
xform.user.username)
)
xform.user = project.organization
xform.save()
except Exception:
self.stdout.write(
"Error processing: {} - {}".format(xform.id_string,
xform.user.username)
)
pass
def handle(self, *args, **options):
self.stdout.write("Re-assigining started", ending='\n')
if not args:
raise CommandError('Param not set. <app model [created_perm]>')
if len(args) < 3:
raise CommandError('Param not set. <app model [created_perm]>')
app = args[0]
model = args[1]
new_perm = args[2]
from onadata.libs.utils.model_tools import queryset_iterator
# Get all the users
for user in queryset_iterator(
User.objects.exclude(pk=settings.ANONYMOUS_USER_ID)):
self.reassign_perms(user, app, model, new_perm)
self.stdout.write("Re-assigining finished", ending='\n')
def _reapply_perms(self, username=None, days=None, seconds=None):
xforms = XForm.objects.none()
the_past = None
if username:
xforms = XForm.objects.filter(user__username=username)
else:
if days:
the_past = timezone.now() - timedelta(days=days)
if seconds:
the_past = timezone.now() - timedelta(seconds=seconds)
if the_past:
xforms = XForm.objects.filter(
Q(date_created__gte=the_past) |
Q(date_modified__gte=the_past))
self.stdout.write(_("{} to be updated").format(xforms.count()))
with use_master:
for xform in queryset_iterator(xforms):
set_project_perms_to_xform(xform, xform.project)
self.stdout.write(gc.collect())
def get_list_of_parsed_instances(self, flat=True):
for i in queryset_iterator(self.instances_for_export(self)):
# TODO: there is information we want to add in parsed xforms.
yield i.get_dict(flat=flat)
def dicts(cls, xform):
qs = cls.objects.filter(instance__xform=xform)
for parsed_instance in queryset_iterator(qs):
yield parsed_instance.to_dict()
def forwards(self, orm):
"Parse all instance to add geoms."
for obj in queryset_iterator(orm['odk_logger.Instance'].objects.all()):
instance = Instance.objects.get(pk=obj.pk)
instance.save(force=True)
def mongo_sync_status(remongo=False, update_all=False, user=None, xform=None):
"""Check the status of records in the mysql db versus mongodb. At a
minimum, return a report (string) of the results.
Optionally, take action to correct the differences, based on these
parameters, if present and defined:
remongo -> if True, update the records missing in mongodb
(default: False)
update_all -> if True, update all the relevant records (default: False)
user -> if specified, apply only to the forms for the given user
(default: None)
xform -> if specified, apply only to the given form (default: None)
"""
qs = XForm.objects.only('id_string', 'user').select_related('user')
if user and not xform:
qs = qs.filter(user=user)
elif user and xform:
qs = qs.filter(user=user, id_string=xform.id_string)
else:
qs = qs.all()
total = qs.count()
found = 0
done = 0
total_to_remongo = 0
report_string = ""
for xform in queryset_iterator(qs, 100):
# get the count
user = xform.user
instance_count = Instance.objects.filter(xform=xform).count()
userform_id = "%s_%s" % (user.username, xform.id_string)
mongo_count = mongo_instances.find(
{common_tags.USERFORM_ID: userform_id}).count()
if instance_count != mongo_count or update_all:
line = "user: %s, id_string: %s\nInstance count: %d\t"\
"Mongo count: %d\n---------------------------------"\
"-----\n" % (
user.username, xform.id_string, instance_count,
mongo_count)
report_string += line
found += 1
total_to_remongo += (instance_count - mongo_count)
# should we remongo
if remongo or (remongo and update_all):
if update_all:
sys.stdout.write(
"Updating all records for %s\n--------------------"
"---------------------------\n" % xform.id_string)
else:
sys.stdout.write(
"Updating missing records for %s\n----------------"
"-------------------------------\n"
% xform.id_string)
update_mongo_for_xform(
xform, only_update_missing=not update_all)
done += 1
sys.stdout.write(
"%.2f %% done ...\r" % ((float(done) / float(total)) * 100))
# only show stats if we are not updating mongo, the update function
# will show progress
if not remongo:
line = "Total # of forms out of sync: %d\n" \
"Total # of records to remongo: %d\n" % (found, total_to_remongo)
report_string += line
return report_string
