def send_email_notice(self):
msgt('Send email notice!')
subject = 'GeoConnect: Clear stale data (%s)' % timezone.now()
self.add_message_line('This is an email notice from Geoconnect',\
prepend=True)
self.add_message_line('-' * 30, prepend=True)
self.add_message_line('-' * 30)
self.add_message_line('(end of message)')
if len(settings.ADMINS) == 0:
msg('No one to email! (no one in settings.ADMINS)')
return
to_addresses = map(lambda x: x[1], settings.ADMINS)
if len(settings.ADMINS) == 0:
msg('No one to email! (no one in settings.ADMINS)')
return
#email_msg = render_to_string('task_scripts/prune_scratch_directories_email.txt', d)
#msg(subject)
#msg(email_msg)
from_email = to_addresses[0]
email_msg = '\n'.join(self.message_lines)
send_mail(subject,
email_msg,
from_email,
to_addresses,
fail_silently=False)
msg('email sent to: %s' % to_addresses)
def test_3_list(self):
"""Save data list to JSONField, check type pre/post"""
msgt(self.test_3_list.__doc__)
#print TestIt.objects.count()
# attributes
#
tname = 'Obj 1'
col_names_val = range(10)
d = dict(name=tname,
column_names=col_names_val)
# make object and save it
#
tunsaved = TestIt(**d)
tunsaved.save()
# Retrieve object
#
tsaved = TestIt.objects.first()
print (tsaved.column_names)
print (type(tsaved.column_names))
self.assertEqual(tsaved.column_names, col_names_val)
self.assertEqual(tsaved.name, tname)
def test_3_list(self):
"""Save data list to JSONField, check type pre/post"""
msgt(self.test_3_list.__doc__)
#print TestIt.objects.count()
# attributes
#
tname = 'Obj 1'
col_names_val = range(10)
d = dict(name=tname, column_names=col_names_val)
# make object and save it
#
tunsaved = TestIt(**d)
tunsaved.save()
# Retrieve object
#
tsaved = TestIt.objects.first()
print tsaved.column_names
print type(tsaved.column_names)
self.assertEqual(tsaved.column_names, col_names_val)
self.assertEqual(tsaved.name, tname)
def test_1_dict(self):
"""Save data dict to JSONField, check type pre/post"""
msgt(self.test_1_dict.__doc__)
# attributes
#
tname = 'Obj 1'
col_names_val = dict(a=1, b=2, c=3) #range(1,10)
d = dict(name=tname,
column_names=col_names_val)
# make object and save it
#
tunsaved = TestIt(**d)
tunsaved.save()
# Retrieve object
#
tsaved = TestIt.objects.first()
print (tsaved.column_names)
print (type(tsaved.column_names))
self.assertEqual(tsaved.column_names, col_names_val)
self.assertEqual(tsaved.name, tname)
def send_email_notice(self):
msgt('Send email notice!')
subject = 'GeoConnect: Clear stale data (%s)' % timezone.now()
self.add_message_line('This is an email notice from Geoconnect',\
prepend=True)
self.add_message_line('-' *30, prepend=True)
self.add_message_line('-' *30)
self.add_message_line('(end of message)')
if len(settings.ADMINS)==0:
msg('No one to email! (no one in settings.ADMINS)')
return
to_addresses = map(lambda x: x[1], settings.ADMINS)
if len(settings.ADMINS)==0:
msg('No one to email! (no one in settings.ADMINS)')
return
#email_msg = render_to_string('task_scripts/prune_scratch_directories_email.txt', d)
#msg(subject)
#msg(email_msg)
from_email = to_addresses[0]
email_msg = '\n'.join(self.message_lines)
send_mail(subject, email_msg, from_email, to_addresses, fail_silently=False)
msg('email sent to: %s' % to_addresses)
def show_instructions():
msgt("quick whitelist check")
msg(('\nRun this command:\n\n >python check_whitelist.py (input file) (output file)'
'\n\n Example:\n\n >python check_whitelist.py whitelist-domains_2017_0405.txt output_2017_0405.csv'
'\n\nThe script may be restarted at a specific line number:'
'\n\n >python check_whitelist.py (input file) (output file) (item #/occurrence in whitelist)'
'\n\n Example. Start at 20th item in the whitelist:'
'\n\n >python check_whitelist.py whitelist-domains_2017_0405.txt output_2017_0405.csv 20'
'\n'))
def remove_stale_dataverse_data(self, stale_age_in_seconds=STALE_AGE_TWO_DAYS):
"""
Here we're removing the metadata and files from dataverse
"""
msgt("Remove stale Dataverse data")
for CLASS_TYPE in (TabularFileInfo, ShapefileInfo):
self.add_message_line('checking: %s' % CLASS_TYPE.__name__)
self.check_for_stale_objects(CLASS_TYPE, stale_age_in_seconds)
self.add_message_line("# of Dataverse objects Checked: %s" % self.num_objects_checked)
self.add_message_line("# of Dataverse objects Removed: %s" % self.num_objects_removed)
def remove_stale_worldmap_data(self, stale_age_in_seconds=DEFAULT_STALE_AGE):
"""
Remove old map data...There are the objects storing WorldMap links
"""
msgt("Remove stale WorldMap data")
for CLASS_TYPE in (WorldMapLatLngInfo, WorldMapJoinLayerInfo, WorldMapShapefileLayerInfo):
self.add_message_line('checking: %s' % CLASS_TYPE.__name__)
self.check_for_stale_objects(CLASS_TYPE, stale_age_in_seconds)
self.add_message_line("# of WorldMap objects Checked: %s" % self.num_objects_checked)
self.add_message_line("# of WorldMap objects Removed: %s" % self.num_objects_removed)
def remove_stale_dataverse_data(self,
stale_age_in_seconds=STALE_AGE_TWO_DAYS):
"""
Here we're removing the metadata and files from dataverse
"""
msgt("Remove stale Dataverse data")
for CLASS_TYPE in (TabularFileInfo, ShapefileInfo):
self.add_message_line('checking: %s' % CLASS_TYPE.__name__)
self.check_for_stale_objects(CLASS_TYPE, stale_age_in_seconds)
self.add_message_line("# of Dataverse objects Checked: %s" %
self.num_objects_checked)
self.add_message_line("# of Dataverse objects Removed: %s" %
self.num_objects_removed)
def remove_stale_worldmap_data(self,
stale_age_in_seconds=DEFAULT_STALE_AGE):
"""
Remove old map data...There are the objects storing WorldMap links
"""
msgt("Remove stale WorldMap data")
for CLASS_TYPE in (WorldMapLatLngInfo, WorldMapJoinLayerInfo,
WorldMapShapefileLayerInfo):
self.add_message_line('checking: %s' % CLASS_TYPE.__name__)
self.check_for_stale_objects(CLASS_TYPE, stale_age_in_seconds)
self.add_message_line("# of WorldMap objects Checked: %s" %
self.num_objects_checked)
self.add_message_line("# of WorldMap objects Removed: %s" %
self.num_objects_removed)
def __init__(self, input_file, output_file, start_num=1, **kwargs):
self.input_file = join(INPUT_DIR, input_file)
self.output_file = join(OUTPUT_DIR, output_file)
try:
self.start_num = int(start_num)
except (ValueError, TypeError) as ex_obj:
msgt("That is not valid start_num (%s)\n%s" % (start_num, ex_obj))
msgx('')
if not isfile(self.input_file):
msgt("Input file not found: %s" % self.input_file)
msgx('')
self.timeout_time = 10 # seconds; add to kwargs
def process_whitelist(self):
"""
Iterate through the white list of domain names
"""
whitelist_names = open(self.input_file, 'r').readlines()
whitelist_names = [x.strip() for x in whitelist_names\
if len(x.strip()) > 0]
fh_out = self.get_output_file_handler()
cnt = 0
for single_name in whitelist_names:
cnt += 1
msgt("(%s) Checking: %s" % (cnt, single_name))
if cnt < self.start_num:
msg('(skipping)')
else:
self.check_single_name(single_name, fh_out)
fh_out.close()
msgt('File written: %s' % self.output_file)