def handle(self, *args, **kwargs):
for i, pi in enumerate(queryset_iterator(ParsedInstance.objects.all())):
pi.update_mongo()
if (i + 1) % 1000 == 0:
print (_('Updated %(nb)d records, flushing MongoDB...')
% {'nb': i})
settings._MONGO_CONNECTION.admin.command({'fsync': 1})
def create_zip_backup(zip_output_file, user, xform):
# create a temp dir that we;ll create our structure within and zip it
# when we are done
tmp_dir_path = tempfile.mkdtemp()
instances_path = os.path.join(tmp_dir_path, "instances")
# get the xls file from storage
# for each submission in the database - create an xml file in this
# form
# /<id_string>/YYYY/MM/DD/YYYY-MM-DD-HH-MM-SS.xml
qs = Instance.objects.filter(xform=xform)
num_instances = qs.count()
done = 0
sys.stdout.write("Creating XML Instances\n")
for instance in queryset_iterator(qs, 100):
# get submission time
date_time_str = instance.date_created.strftime(DATE_FORMAT)
date_parts = date_time_str.split("-")
sub_dirs = os.path.join(*date_parts[:3])
# create the directories
full_path = os.path.join(instances_path, sub_dirs)
if not os.path.exists(full_path):
try:
os.makedirs(full_path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
full_xml_path = os.path.join(full_path, date_time_str + ".xml")
# check for duplicate file names
file_index = 1
while os.path.exists(full_xml_path):
full_xml_path = os.path.join(full_path, "%s-%d.xml" % (date_time_str, file_index))
file_index += 1
# create the instance xml
with codecs.open(full_xml_path, "wb", "utf-8") as f:
f.write(instance.xml)
done += 1
sys.stdout.write("\r%.2f %% done" % (float(done) / float(num_instances) * 100))
sys.stdout.flush()
sleep(0)
# write zip file
sys.stdout.write("\nWriting to ZIP arhive.\n")
zf = zipfile.ZipFile(zip_output_file, "w")
done = 0
for dir_path, dir_names, file_names in os.walk(tmp_dir_path):
for file_name in file_names:
archive_path = dir_path.replace(tmp_dir_path + os.path.sep, "", 1)
zf.write(os.path.join(dir_path, file_name), os.path.join(archive_path, file_name))
done += 1
sys.stdout.write("\r%.2f %% done" % (float(done) / float(num_instances) * 100))
sys.stdout.flush()
sleep(0)
zf.close()
# removed dir tree
shutil.rmtree(tmp_dir_path)
sys.stdout.write("\nBackup saved to %s\n" % zip_output_file)
def handle(self, *args, **kwargs):
for i, pi in enumerate(queryset_iterator(
ParsedInstance.objects.all())):
pi.update_mongo()
if (i + 1) % 1000 == 0:
print 'Updated %d records, flushing MongoDB...' % i
settings._MONGO_CONNECTION.admin.command({'fsync': 1})
def handle(self, *args, **kwargs):
print "%d XForms to update" % DataDictionary.objects.count()
for i, dd in enumerate(queryset_iterator(DataDictionary.objects.all())):
if dd.xls:
dd._set_uuid_in_xml()
super(DataDictionary, dd).save()
if (i + 1) % 10 == 0:
print "Updated %d XForms..." % i
def handle(self, *args, **kwargs):
print (_('%(nb)d XForms to update')
% {'nb': DataDictionary.objects.count()})
for i, dd in enumerate(queryset_iterator(DataDictionary.objects.all())):
if dd.xls:
dd._set_uuid_in_xml()
super(DataDictionary, dd).save()
if (i + 1) % 10 == 0:
print _('Updated %(nb)d XForms...') % {'nb': i}
def mongo_sync_status(remongo=False, update_all=False, user=None, xform=None):
qs = XForm.objects.only('id_string').select_related('user')
if user and not xform:
qs = qs.filter(user=user)
elif user and xform:
qs = qs.filter(user=user, id_string=xform.id_string)
else:
qs = qs.all()
total = qs.count()
found = 0
done = 0
total_to_remongo = 0
report_string = ""
for xform in queryset_iterator(qs, 100):
# get the count
user = xform.user
instance_count = Instance.objects.filter(xform=xform).count()
userform_id = "%s_%s" % (user.username, xform.id_string)
mongo_count = mongo_instances.find(
{common_tags.USERFORM_ID: userform_id}).count()
if instance_count != mongo_count or update_all:
line = "user: %s, id_string: %s\nInstance count: %d\t"\
"Mongo count: %d\n---------------------------------"\
"-----\n" % (
user.username, xform.id_string, instance_count,
mongo_count)
report_string += line
found += 1
total_to_remongo += (instance_count - mongo_count)
# should we remongo
if remongo or (remongo and update_all):
if update_all:
sys.stdout.write(
"Updating all records for %s\n--------------------"
"---------------------------\n" % xform.id_string)
else:
sys.stdout.write(
"Updating missing records for %s\n----------------"
"-------------------------------\n"
% xform.id_string)
update_mongo_for_xform(
xform, only_update_missing=not update_all)
done += 1
sys.stdout.write(
"%.2f %% done ...\r" % ((float(done) / float(total)) * 100))
# only show stats if we are not updating mongo, the update function
# will show progress
if not remongo:
line = "Total # of forms out of sync: %d\n" \
"Total # of records to remongo: %d\n" % (
found, total_to_remongo)
report_string += line
return report_string
def mongo_sync_status(remongo=False, update_all=False, user=None, xform=None):
qs = XForm.objects.only('id_string', 'user').select_related('user')
if user and not xform:
qs = qs.filter(user=user)
elif user and xform:
qs = qs.filter(user=user, id_string=xform.id_string)
else:
qs = qs.all()
total = qs.count()
found = 0
done = 0
total_to_remongo = 0
report_string = ""
for xform in queryset_iterator(qs, 100):
# get the count
user = xform.user
instance_count = Instance.objects.filter(xform=xform).count()
userform_id = "%s_%s" % (user.username, xform.id_string)
mongo_count = mongo_instances.find({
common_tags.USERFORM_ID: userform_id
}).count()
if instance_count != mongo_count or update_all:
line = "user: %s, id_string: %s\nInstance count: %d\t"\
"Mongo count: %d\n---------------------------------"\
"-----\n" % (
user.username, xform.id_string, instance_count,
mongo_count)
report_string += line
found += 1
total_to_remongo += (instance_count - mongo_count)
# should we remongo
if remongo or (remongo and update_all):
if update_all:
sys.stdout.write(
"Updating all records for %s\n--------------------"
"---------------------------\n" % xform.id_string)
else:
sys.stdout.write(
"Updating missing records for %s\n----------------"
"-------------------------------\n" % xform.id_string)
update_mongo_for_xform(xform,
only_update_missing=not update_all)
done += 1
sys.stdout.write("%.2f %% done ...\r" %
((float(done) / float(total)) * 100))
# only show stats if we are not updating mongo, the update function
# will show progress
if not remongo:
line = "Total # of forms out of sync: %d\n" \
"Total # of records to remongo: %d\n" % (
found, total_to_remongo)
report_string += line
return report_string
def handle(self, *args, **kwargs):
attachments_qs = Attachment.objects.select_related(
'instance', 'instance__xform')
if kwargs.get('username'):
username = kwargs.get('username')
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
raise CommandError(
"Error: username %(username)s does not exist" %
{'username': username})
attachments_qs = attachments_qs.filter(instance__user=user)
if kwargs.get('id_string'):
id_string = kwargs.get('id_string')
try:
xform = XForm.objects.get(id_string=id_string)
except XForm.DoesNotExist:
raise CommandError(
"Error: Form with id_string %(id_string)s does not exist" %
{'id_string': id_string})
attachments_qs = attachments_qs.filter(instance__xform=xform)
fs = get_storage_class('django.core.files.storage.FileSystemStorage')()
for att in queryset_iterator(attachments_qs):
filename = att.media_file.name
default_storage = get_storage_class()()
full_path = get_path(filename,
settings.THUMB_CONF['small']['suffix'])
if kwargs.get('force') is not None:
for s in ['small', 'medium', 'large']:
fp = get_path(filename, settings.THUMB_CONF[s]['suffix'])
if default_storage.exists(fp):
default_storage.delete(fp)
if not default_storage.exists(full_path):
try:
if default_storage.__class__ != fs.__class__:
resize(filename)
else:
resize_local_env(filename)
if default_storage.exists(
get_path(
filename, '%s' %
settings.THUMB_CONF['small']['suffix'])):
print(
_(u'Thumbnails created for %(file)s') %
{'file': filename})
else:
print(
_(u'Problem with the file %(file)s') %
{'file': filename})
except (IOError, OSError), e:
print _(u'Error on %(filename)s: %(error)s') \
% {'filename': filename, 'error': e}
def handle(self, *args, **kwargs):
attachments_qs = Attachment.objects.select_related(
'instance', 'instance__xform')
if kwargs.get('username'):
username = kwargs.get('username')
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
raise CommandError(
"Error: username %(username)s does not exist" %
{'username': username}
)
attachments_qs = attachments_qs.filter(instance__user=user)
if kwargs.get('id_string'):
id_string = kwargs.get('id_string')
try:
xform = XForm.objects.get(id_string=id_string)
except XForm.DoesNotExist:
raise CommandError(
"Error: Form with id_string %(id_string)s does not exist" %
{'id_string': id_string}
)
attachments_qs = attachments_qs.filter(instance__xform=xform)
fs = get_storage_class('django.core.files.storage.FileSystemStorage')()
for att in queryset_iterator(attachments_qs):
filename = att.media_file.name
default_storage = get_storage_class()()
full_path = get_path(filename,
settings.THUMB_CONF['small']['suffix'])
if kwargs.get('force') is not None:
for s in ['small', 'medium', 'large']:
fp = get_path(filename,
settings.THUMB_CONF[s]['suffix'])
if default_storage.exists(fp):
default_storage.delete(fp)
if not default_storage.exists(full_path):
try:
if default_storage.__class__ != fs.__class__:
resize(filename)
else:
resize_local_env(filename)
if default_storage.exists(get_path(
filename,
'%s' % settings.THUMB_CONF['small']['suffix'])):
print (_(u'Thumbnails created for %(file)s')
% {'file': filename})
else:
print (_(u'Problem with the file %(file)s')
% {'file': filename})
except (IOError, OSError), e:
print _(u'Error on %(filename)s: %(error)s') \
% {'filename': filename, 'error': e}
def handle(self, *args, **kwargs):
xforms = XForm.objects.all()
total = xforms.count()
count = 0
for xform in queryset_iterator(XForm.objects.all()):
has_geo = ParsedInstance.objects.filter(
instance__xform=xform, lat__isnull=False).count() > 0
try:
xform.surveys_with_geopoints = has_geo
xform.save()
except Exception as e:
print e
else:
count += 1
print "%d of %d forms processed." % (count, total)
def handle(self, *args, **kwargs):
fs = get_storage_class('django.core.files.storage.FileSystemStorage')()
for att in queryset_iterator(Attachment.objects.select_related(
'instance', 'instance__xform').all()):
filename = att.media_file.name
default_storage = get_storage_class()()
if not default_storage.exists(get_path(filename,
settings.THUMB_CONF['smaller']['suffix'])):
try:
if default_storage.__class__ != fs.__class__:
resize(filename)
else:
resize_local_env(filename)
if default_storage.exists(get_path(filename,
settings.THUMB_CONF['smaller']['suffix'])):
print 'Thumbnails created for %s' % filename
else:
print 'Something didn\'t go right for %s' % filename
except (IOError, OSError), e:
print 'Error on %s: %s' % (filename, e)
def handle(self, *args, **kwargs):
fs = get_storage_class('django.core.files.storage.FileSystemStorage')()
for att in queryset_iterator(Attachment.objects.select_related(
'instance', 'instance__xform').all()):
filename = att.media_file.name
default_storage = get_storage_class()()
if not default_storage.exists(get_path(filename,
settings.THUMB_CONF['smaller']['suffix'])):
try:
if default_storage.__class__ != fs.__class__:
resize(filename)
else:
resize_local_env(filename)
if default_storage.exists(get_path(filename,
'%s.%s' % (settings.THUMB_CONF['smaller']['suffix'],
settings.IMG_FILE_TYPE))):
print _(u'Thumbnails created for %s') % filename
else:
print _(u'Problem with the file %s') % filename
except (IOError, OSError), e:
print _(u'Error on %(filename)s: %(error)s') \
% {'filename': filename, 'error': e}
def handle(self, *args, **kwargs):
fs = get_storage_class('django.core.files.storage.FileSystemStorage')()
for att in queryset_iterator(
Attachment.objects.select_related('instance',
'instance__xform').all()):
filename = att.media_file.name
default_storage = get_storage_class()()
if not default_storage.exists(
get_path(filename,
settings.THUMB_CONF['smaller']['suffix'])):
try:
if default_storage.__class__ != fs.__class__:
resize(filename)
else:
resize_local_env(filename)
if default_storage.exists(
get_path(
filename,
settings.THUMB_CONF['smaller']['suffix'])):
print 'Thumbnails created for %s' % filename
else:
print 'Something didn\'t go right for %s' % filename
except (IOError, OSError), e:
print 'Error on %s: %s' % (filename, e)
def dicts(cls, xform):
qs = cls.objects.filter(instance__xform=xform)
for parsed_instance in queryset_iterator(qs):
yield parsed_instance.to_dict()
def dicts(cls, xform):
qs = cls.objects.filter(instance__xform=xform)
for parsed_instance in queryset_iterator(qs):
yield parsed_instance.to_dict()
def handle(self, *args, **kwargs):
user = xform = None
if len(args) > 0:
username = args[0]
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
raise CommandError("User %s does not exist" % username)
if len(args) > 1:
id_string = args[1]
try:
xform = XForm.objects.get(user=user, id_string=id_string)
except XForm.DoesNotExist:
raise CommandError(
"Xform %s does not exist for user %s" %\
(id_string, user.username))
remongo = kwargs["remongo"]
update_all = kwargs["update_all"]
qs = XForm.objects.only('id_string').select_related('user')
if user and not xform:
qs = qs.filter(user=user)
elif user and xform:
qs = qs.filter(user=user, id_string=xform.id_string)
else:
qs = qs.all()
total = qs.count()
found = 0
done = 0
total_to_remongo = 0
for xform in queryset_iterator(qs, 100):
# get the count
user = xform.user
instance_count = Instance.objects.filter(xform=xform).count()
userform_id = "%s_%s" % (user.username, xform.id_string)
mongo_count = xform_instances.find(
{"_userform_id": userform_id}).count()
if instance_count != mongo_count or update_all:
line = "user: %s, id_string: %s\nInstance count: %d\t" \
"Mongo count: %d\n---------------------------------" \
"-----\n" % (
user.username, xform.id_string, instance_count,
mongo_count)
self.stdout.write(line)
found += 1
total_to_remongo += (instance_count - mongo_count)
# should we remongo
if remongo or (remongo and update_all):
if update_all:
self.stdout.write(
"Updating all records for %s\n--------------------"
"---------------------------\n" % xform.id_string)
else:
self.stdout.write(
"Updating missing records for %s\n----------------"
"-------------------------------\n" %\
xform.id_string)
update_mongo_for_xform(
xform, only_update_missing=not update_all)
done += 1
self.stdout.write(
"%.2f %% done ...\r" % ((float(done)/float(total)) * 100))
# only show stats if we are not updating mongo, the update function
# will show progress
if not remongo:
line = "Total Forms out of sync: %d\nTotal to remongo: %d\n" % (
found, total_to_remongo)
self.stdout.write(line)
def get_list_of_parsed_instances(self, flat=True):
for i in queryset_iterator(self.surveys_for_export(self)):
# TODO: there is information we want to add in parsed xforms.
yield i.get_dict(flat=flat)
def mongo_sync_status(remongo=False, update_all=False, user=None, xform=None):
"""Check the status of records in the mysql db versus mongodb. At a
minimum, return a report (string) of the results.
Optionally, take action to correct the differences, based on these
parameters, if present and defined:
remongo -> if True, update the records missing in mongodb (default: False)
update_all -> if True, update all the relevant records (default: False)
user -> if specified, apply only to the forms for the given user (default: None)
xform -> if specified, apply only to the given form (default: None)
"""
qs = XForm.objects.only('id_string', 'user').select_related('user')
if user and not xform:
qs = qs.filter(user=user)
elif user and xform:
qs = qs.filter(user=user, id_string=xform.id_string)
else:
qs = qs.all()
total = qs.count()
found = 0
done = 0
total_to_remongo = 0
report_string = ""
for xform in queryset_iterator(qs, 100):
# get the count
user = xform.user
instance_count = Instance.objects.filter(xform=xform).count()
userform_id = "%s_%s" % (user.username, xform.id_string)
mongo_count = mongo_instances.find({
common_tags.USERFORM_ID: userform_id
}).count()
if instance_count != mongo_count or update_all:
line = "user: %s, id_string: %s\nInstance count: %d\t"\
"Mongo count: %d\n---------------------------------"\
"-----\n" % (
user.username, xform.id_string, instance_count,
mongo_count)
report_string += line
found += 1
total_to_remongo += (instance_count - mongo_count)
# should we remongo
if remongo or (remongo and update_all):
if update_all:
sys.stdout.write(
"Updating all records for %s\n--------------------"
"---------------------------\n" % xform.id_string)
else:
sys.stdout.write(
"Updating missing records for %s\n----------------"
"-------------------------------\n" % xform.id_string)
update_mongo_for_xform(xform,
only_update_missing=not update_all)
done += 1
sys.stdout.write("%.2f %% done ...\r" %
((float(done) / float(total)) * 100))
# only show stats if we are not updating mongo, the update function
# will show progress
if not remongo:
line = "Total # of forms out of sync: %d\n" \
"Total # of records to remongo: %d\n" % (
found, total_to_remongo)
report_string += line
return report_string
def create_zip_backup(zip_output_file, user, xform):
# create a temp dir that we;ll create our structure within and zip it
# when we are done
tmp_dir_path = tempfile.mkdtemp()
instances_path = os.path.join(tmp_dir_path, "instances")
# get the xls file from storage
# for each submission in the database - create an xml file in this
# form
# /<id_string>/YYYY/MM/DD/YYYY-MM-DD-HH-MM-SS.xml
qs = Instance.objects.filter(xform=xform)
num_instances = qs.count()
done = 0
sys.stdout.write("Creating XML Instances\n")
for instance in queryset_iterator(qs, 100):
# get submission time
date_time_str = instance.date_created.strftime(DATE_FORMAT)
date_parts = date_time_str.split("-")
sub_dirs = os.path.join(*date_parts[:3])
# create the directories
full_path = os.path.join(instances_path, sub_dirs)
if not os.path.exists(full_path):
try:
os.makedirs(full_path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
full_xml_path = os.path.join(full_path, date_time_str + ".xml")
# check for duplicate file names
file_index = 1
while os.path.exists(full_xml_path):
full_xml_path = os.path.join(
full_path, "%s-%d.xml" % (date_time_str, file_index))
file_index += 1
# create the instance xml
with codecs.open(full_xml_path, "wb", "utf-8") as f:
f.write(instance.xml)
done += 1
sys.stdout.write("\r%.2f %% done" %
(float(done) / float(num_instances) * 100))
sys.stdout.flush()
sleep(0)
# write zip file
sys.stdout.write("\nWriting to ZIP arhive.\n")
zf = zipfile.ZipFile(zip_output_file, "w")
done = 0
for dir_path, dir_names, file_names in os.walk(tmp_dir_path):
for file_name in file_names:
archive_path = dir_path.replace(tmp_dir_path + os.path.sep, "", 1)
zf.write(os.path.join(dir_path, file_name),
os.path.join(archive_path, file_name))
done += 1
sys.stdout.write("\r%.2f %% done" %
(float(done) / float(num_instances) * 100))
sys.stdout.flush()
sleep(0)
zf.close()
# removed dir tree
shutil.rmtree(tmp_dir_path)
sys.stdout.write("\nBackup saved to %s\n" % zip_output_file)
def get_list_of_parsed_instances(self, flat=True):
for i in queryset_iterator(self.surveys_for_export(self)):
# TODO: there is information we want to add in parsed xforms.
yield i.get_dict(flat=flat)
def mongo_sync_status(remongo=False, update_all=False, user=None, xform=None):
"""Check the status of records in the mysql db versus mongodb. At a
minimum, return a report (string) of the results.
Optionally, take action to correct the differences, based on these
parameters, if present and defined:
remongo -> if True, update the records missing in mongodb (default: False)
update_all -> if True, update all the relevant records (default: False)
user -> if specified, apply only to the forms for the given user (default: None)
xform -> if specified, apply only to the given form (default: None)
"""
qs = XForm.objects.only('id_string', 'user').select_related('user')
if user and not xform:
qs = qs.filter(user=user)
elif user and xform:
qs = qs.filter(user=user, id_string=xform.id_string)
else:
qs = qs.all()
total = qs.count()
found = 0
done = 0
total_to_remongo = 0
report_string = ""
for xform in queryset_iterator(qs, 100):
# get the count
user = xform.user
instance_count = Instance.objects.filter(xform=xform).count()
userform_id = "%s_%s" % (user.username, xform.id_string)
mongo_count = mongo_instances.find(
{common_tags.USERFORM_ID: userform_id}).count()
if instance_count != mongo_count or update_all:
line = "user: %s, id_string: %s\nInstance count: %d\t"\
"Mongo count: %d\n---------------------------------"\
"-----\n" % (
user.username, xform.id_string, instance_count,
mongo_count)
report_string += line
found += 1
total_to_remongo += (instance_count - mongo_count)
# should we remongo
if remongo or (remongo and update_all):
if update_all:
sys.stdout.write(
"Updating all records for %s\n--------------------"
"---------------------------\n" % xform.id_string)
else:
sys.stdout.write(
"Updating missing records for %s\n----------------"
"-------------------------------\n"
% xform.id_string)
update_mongo_for_xform(
xform, only_update_missing=not update_all)
done += 1
sys.stdout.write(
"%.2f %% done ...\r" % ((float(done) / float(total)) * 100))
# only show stats if we are not updating mongo, the update function
# will show progress
if not remongo:
line = "Total # of forms out of sync: %d\n" \
"Total # of records to remongo: %d\n" % (
found, total_to_remongo)
report_string += line
return report_string
