def handle(self, *args, **options):
Sighting.objects.all().delete()
with open(options['csv_file']) as fp:
reader = csv.DictReader(fp)
rows = list(reader)
for row in rows:
s = Sighting(
Longitude=row['X'],
Latitude=row['Y'],
Unique_Squirrel_Id=row['Unique Squirrel ID'],
Shift=row['Shift'],
Date=datetime.datetime.strptime(row['Date'], '%m%d%Y'),
Age=row['Age'],
Primary_Fur_Color=row['Primary Fur Color'],
Location=row['Location'],
Specific_Location=row['Specific Location'],
Running=strtobool(row['Running']),
Chasing=strtobool(row['Chasing']),
Climbing=strtobool(row['Climbing']),
Eating=strtobool(row['Eating']),
Foraging=strtobool(row['Foraging']),
Other_Activities=row['Other Activities'],
Kuks=strtobool(row['Kuks']),
Quaas=strtobool(row['Quaas']),
Moans=strtobool(row['Moans']),
Tail_Flags=strtobool(row['Tail flags']),
Tail_Twitches=strtobool(row['Tail twitches']),
Approaches=strtobool(row['Approaches']),
Indifferent=strtobool(row['Indifferent']),
Runs_From=strtobool(row['Runs from']),
)
s.save()
def handle(self, *args, **kwargs):
with open(kwargs['path'][0]) as file:
reader = csv.DictReader(file)
data = list(reader)
for item in data:
s = Sighting(
x = item['x'],
y = item['y'],
unique_squirrel_id = item['unique_squirrel_id'],
shift = item['shift'],
date = item['date'],
age = item['age'],
primary_fur_color = item['primary_fur_color'],
location = item['location'],
specific_location = item['specific_location'],
running = item['running'],
climbing = item['climbing'],
eating = item['eating'],
foraging = item['foraging'],
other_activities = item['other_activities'],
kuks = item['kuks'],
quaas = item['quaas'],
moans = item['moans'],
tail_flags = item['tail_flags'],
tail_twitches = item['tail_twitches'],
approaches = item['approaches'],
indifferent = item['indifferent'],
runs_from = item['runs_from'],
)
s.save()
def handle(self, *args, **options):
with open(options['csv_file']) as fp:
reader = csv.DictReader(fp)
data = list(reader)
for item in data:
sighting = Sighting(
latitude=item['Y'],
longitude=item['X'],
uid=item['Unique Squirrel ID'],
shift=item['Shift'],
date=datetime.strptime(item['Date'], '%m%d%Y').date(),
age='' if item['Age'] == '?' else item['Age'],
color=item['Primary Fur Color'],
loc=item['Location'],
specific_loc=item['Specific Location'],
running=convert(item['Running']),
chasing=convert(item['Chasing']),
climbing=convert(item['Climbing']),
eating=convert(item['Eating']),
foraging=convert(item['Foraging']),
other_act=convert(item['Other Activities']),
kuks=convert(item['Kuks']),
quaas=convert(item['Quaas']),
moans=convert(item['Moans']),
tail_flags=convert(item['Tail flags']),
tail_twitches=convert(item['Tail twitches']),
approaches=convert(item['Approaches']),
indifferent=convert(item['Indifferent']),
runs_from=convert(item['Runs from']),
)
sighting.save()
def post_sighting(request):
if request.method == 'POST':
searchquery = json.loads(request.body)
speciesItem = Species.objects.get(
specname__contains=searchquery['species'])
#speciesPK = speciesItem.pk
sighting = Sighting(sub_date=searchquery['date'],
species=speciesItem,
animals=searchquery['animals'],
location=searchquery['location'],
latitude=searchquery['lat'],
longitude=searchquery['lng'],
name=searchquery['name'],
imageurl=searchquery['imageurl'])
sighting.save()
data_to_dump = {'success': 'success'}
data = simplejson.dumps(data_to_dump)
return HttpResponse(data, mimetype='application/json')
#result = serializers.serialize('json', Sighting.objects.all(), use_natural_keys=True)
#return HttpResponse(result, mimetype='application/json')
else:
data_to_dump = {'success': 'failure'}
data = simplejson.dumps(data_to_dump)
return HttpResponse(data, mimetype='application/json')
def handle(self, *args, **options):
filepath = options['csvfile'][0]
import os
import csv
#import datetime
pattern = re.compile(r'(\d{2})(\d{2})(\d{4})')
with open(filepath, 'r') as csvFile:
reader = csv.DictReader(csvFile, delimiter=',')
list_ = []
count = 0
for line in reader:
# line['Date'] = datetime.datetime.strptime(line['Date'],'%m%d%Y')
# l = reader.line_num # Lines start numbering at 1, not 0
# if line == []:
# line = next(reader) # Check for empty lines
# if l == 1:
# header = line # Get field names
# line = next(reader) # Move on to the first line of data
# while line['Unique Squirrel ID'] in Sighting.objects.values_list('Unique_Squirrel_ID', flat=True):
# line['Unique Squirrel ID'] += '-R'
m, d, y = re.match(pattern, line['Date']).groups()
if line['Unique Squirrel ID'] in list_:
continue
else:
list_.append(line['Unique Squirrel ID'])
s = Sighting(X=line['X'],
Y=line['Y'],
unique_squirrel_id=line['Unique Squirrel ID'],
Shift=line['Shift'],
Date=date(int(y), int(m), int(d)),
Age=line['Age'],
Primary_Fur_Color=line['Primary Fur Color'],
Location=line['Location'],
Specific_Location=line['Specific Location'],
Running=line['Running'],
Chasing=line['Chasing'],
Climbing=line['Climbing'],
Eating=line['Eating'],
Foraging=line['Foraging'],
Other_Activities=line['Other Activities'],
Kuks=line['Kuks'],
Quaas=line['Quaas'],
Moans=line['Moans'],
Tail_Flags=line['Tail flags'],
Tail_Twitches=line['Tail twitches'],
Approaches=line['Approaches'],
Indifferent=line['Indifferent'],
Runs_From=line['Runs from'])
s.save()
count = count + 1
print(count)
def handle(self, *args, **options):
with open(options['csv_file']) as fp:
reader = csv.DictReader(fp)
data = list(reader)
sightings = []
for dict_ in data:
sightings.append(
Sighting(
latitude=float(dict_['Y']),
longitude=float(dict_['X']),
unique_squirrel_id=dict_['Unique Squirrel ID'],
shift=dict_['Shift'].lower(),
date=timezone.datetime.strptime(dict_['Date'],
'%m%d%Y').date(),
age=dict_['Age'],
primary_fur_color=dict_['Primary Fur Color'],
specific_location=dict_['Specific Location'],
running=dict_['Running'].lower() == 'true',
chasing=dict_['Chasing'].lower() == 'true',
climbing=dict_['Climbing'].lower() == 'true',
eating=dict_['Eating'].lower() == 'true',
foraging=dict_['Foraging'].lower() == 'true',
other_activities=dict_['Other Activities'],
kuks=dict_['Kuks'].lower() == 'true',
quaas=dict_['Quaas'].lower() == 'true',
moans=dict_['Moans'].lower() == 'true',
tail_flags=dict_['Tail flags'].lower() == 'true',
tail_twitches=dict_['Tail twitches'].lower() == 'true',
approaches=dict_['Approaches'].lower() == 'true',
indifferent=dict_['Indifferent'].lower() == 'true',
runs_from=dict_['Runs from'].lower() == 'true',
))
Sighting.objects.bulk_create(sightings)
def test_media_creation(self):
in_caracas = Sighting.get(Sighting.lat == 80.08)
picture = Media(filename='test.file',
content_type='application/unknown',
url='www.test_file.com',
sighting=in_caracas)
picture.save()
in_caracas.update()
assert in_caracas.media_files[0].id == picture.id
def post_sighting(request):
if request.method == 'POST':
searchquery = json.loads(request.body)
speciesItem = Species.objects.get(specname__contains=searchquery['species'])
#speciesPK = speciesItem.pk
sighting = Sighting(sub_date=searchquery['date'], species=speciesItem, animals=searchquery['animals'],
location=searchquery['location'], latitude=searchquery['lat'], longitude=searchquery['lng'],
name=searchquery['name'], imageurl=searchquery['imageurl'])
sighting.save()
data_to_dump = {'success': 'success'}
data = simplejson.dumps(data_to_dump)
return HttpResponse(data, mimetype='application/json')
#result = serializers.serialize('json', Sighting.objects.all(), use_natural_keys=True)
#return HttpResponse(result, mimetype='application/json')
else:
data_to_dump = {'success': 'failure'}
data = simplejson.dumps(data_to_dump)
return HttpResponse(data, mimetype='application/json')
def handle(self,*arg,**options):
import csv
import datetime
from sightings.models import Sighting
path=str(options['csv_file'][0])
with open(path) as f:
data=csv.reader(f)
next(data)
counter=0
for line in data:
for i in (15,16,17,18,19,21,22,23,24,25,26,27,28):
if line[i]=='false':
line[i]= False
else:
line[i]= True
line[5]=datetime.datetime.strptime(line[5],"%m%d%Y").strftime("%Y-%m-%d")
sighting= Sighting(latitude=line[1],
longitude=line[0],
squirrel_id=line[2],
shift=line[4],
date=line[5],
age=line[7],
fur_color=line[8],
location=line[12],
specific_location=line[14],
running=line[15],
chasing=line[16],
climbing=line[17],
eating=line[18],
foraging=line[19],
other_activities=line[20],
kuks=line[21],
quaas=line[22],
moans=line[23],
tail_flags=line[24],
tail_twitches=line[25],
approaches=line[26],
indifferent=line[27],
runs_from=line[28],)
#try:
sighting.save()
def handle(self, *arg, **options):
path = str(options['csv_file'][0])
with open(path) as csvfile:
data = csv.reader(csvfile)
next(data)
counter = 0
for row in data:
for i in (15, 16, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28):
if row[i] == 'true':
row[i] = True
else:
row[i] = False
row[5] = datetime.datetime.strptime(
row[5], "%m%d%Y").strftime("%Y-%m-%d")
sighting = Sighting(
longitude=row[0],
latitude=row[1],
squirrel_id=row[2],
shift=row[4],
date=row[5],
age=row[7],
primary_fur_color=row[8],
location=row[12],
specific_location=row[14],
running=row[15],
chasing=row[16],
climbing=row[17],
eating=row[18],
foraging=row[19],
other_activities=row[20],
kuks=row[21],
quaas=row[22],
moans=row[23],
tail_flags=row[24],
tail_twitches=row[25],
approaches=row[26],
indifferent=row[27],
runs_from=row[28],
)
sighting.save()
def handle(self, *args, **options):
filepath = options['csvfile'][0]
import os
import csv
pattern = re.compile(r'(\d{2})(\d{2})(\d{4})')
with open(filepath, 'r') as csvFile:
reader = csv.DictReader(csvFile, delimiter=',')
list_ = []
count = 0
for line in reader:
m,d,y = re.match(pattern, line['Date']).groups()
if line['Unique Squirrel ID'] in list_:
continue
else:
list_.append(line['Unique Squirrel ID'])
s = Sighting(X=line['X'],
Y=line['Y'],
unique_squirrel_id=line['Unique Squirrel ID'],
Shift=line['Shift'],
Date=date(int(y),int(m),int(d)),
Age = line['Age'],
Primary_Fur_Color = line['Primary Fur Color'],
Location = line['Location'],
Specific_Location = line['Specific Location'],
Running = line['Running'],
Chasing = line['Chasing'],
Climbing = line['Climbing'],
Eating = line['Eating'],
Foraging = line['Foraging'],
Other_Activities = line['Other Activities'],
Kuks = line['Kuks'],
Quaas = line['Quaas'],
Moans = line['Moans'],
Tail_Flags = line['Tail flags'],
Tail_Twitches = line['Tail twitches'],
Approaches = line['Approaches'],
Indifferent = line['Indifferent'],
Runs_From = line['Runs from'])
s.save()
count = count+1
print(count)
def test_sighting_creation(self):
in_caracas = Sighting(lat=80.08, long=99.99)
in_caracas.save()
luis = Subject.get(Subject.name == 'Luis')
rita = Subject.get(Subject.name == 'Rita')
in_caracas.subjects = [luis, rita]
in_caracas.save()
luis.update()
assert len(luis.sightings) == 1
def handle(self, *args, **options):
data = pd.read_csv(options['file_path'])
data.columns = [
c.lower().replace(' ', '_').replace('/', '_') for c in data.columns
]
data.rename(columns={"x": "latitude", "y": "longitude"}, inplace=True)
data['date'] = pd.to_datetime(data['date'].astype(str),
format='%m%d%Y')
sightList = data.to_dict(orient='records')
sightList = [Sighting(**s) for s in sightList]
Sighting.objects.all().delete()
Sighting.objects.bulk_create(sightList)
def handle(self,*args,**options):
path = kwargs['path']
with open(path, 'r') as file:
rows = csv.reader(file)
header = next(rows)
for row in rows:
t = row[5]
s = Sighting(
longitude = row[0],
latitude = row[1],
unique_squirrel_id = row[2],
shift = row[4],
date = timezone.datetime(int(t[4:]),int(t[:2]),int(t[2:4])).date(),
age = row[7],
primary_fur_color = row[8],
location = row[12],
specific_location = row[14],
running = str2bool(row[15]),
chasing = str2bool(row[16]),
climbing = str2bool(row[17]),
eating = str2bool(row[18]),
foraging = str2bool(row[19]),
other_activities = row[20],
kuks = str2bool(row[21]),
quaas = str2bool(row[22]),
moans = str2bool(row[23]),
tail_flags = str2bool(row[24]),
tail_twitches = str2bool(row[25]),
approaches = str2bool(row[26]),
indifferent = str2bool(row[27]),
runs_from = str2bool(row[28]),
)
s.save()
file.close()
self.stdout.write(self.style.SUCCESS("Successfully upload squirrel data!"))
def handle(self, *args, **options):
def func(x):
if x.lower() == 'true':
return 'True'
else:
return 'False'
with open(options['csv_file']) as fp:
reader = csv.DictReader(fp)
data = list(reader)
for row in data:
s = Sighting(latitude = float(row['Y']),
longitude = float(row['X']),
unique_squirrel_id = row['Unique Squirrel ID'],
shift = row['Shift'],
date = datetime.datetime.strptime(row['Date'],'%m%d%Y'),
age = row['Age'].capitalize(),
primary_fur_color = row['Primary Fur Color'].capitalize(),
location = row['Location'].capitalize(),
specific_location = row['Specific Location'].capitalize(),
running = func(row['Running'].capitalize()),
chasing = func(row['Chasing'].capitalize()),
climbing = func(row['Climbing'].capitalize()),
eating = func(row['Eating'].capitalize()),
foraging = func(row['Foraging'].capitalize()),
other_activities = row['Other Activities'].capitalize(),
kuks = func(row['Kuks'].capitalize()),
quaas = func(row['Quaas'].capitalize()),
moans = func(row['Moans'].capitalize()),
tail_flags =func(row['Tail flags'].capitalize()),
tail_twitches = func(row['Tail twitches'].capitalize()),
approaches = func(row['Approaches'].capitalize()),
indifferent = func(row['Indifferent'].capitalize()),
runs_from = func(row['Runs from'].capitalize()),
)
s.save()
def handle(self, *args, **options):
with open(options['squirrel_csv']) as fp:
reader = csv.DictReader(fp)
noduplicates = list()
for item in reader:
if item['Unique Squirrel ID'] in noduplicates:
continue
else:
obj = Sighting()
obj.longitude = float(item['X']),
obj.latitude = float(item['Y']),
obj.unique_squirrel_id = item['Unique Squirrel ID'],
obj.shift = item['Shift'],
obj.date = timezone.datetime.strptime(str(item['Date']), '%m%d%Y').date(),
obj.age = item['Age'],
obj.primary_fur_color = item['Primary Fur Color'],
obj.location = item['Location'],
obj.specific_location = item['Specific Location'],
obj.running = item['Running'] == 'TRUE',
obj.chasing = item['Chasing'] == 'TRUE',
obj.climbing = item['Climbing'] == 'TRUE',
obj.eating = item['Eating'] == 'TRUE',
obj.foraging = item['Foraging'] == 'TRUE',
obj.other_activities = item['Other Activities'],
obj.kuks = item['Kuks'] == 'TRUE',
obj.quaas = item['Quaas'] == 'TRUE',
obj.moans = item['Moans'] == 'TRUE',
obj.tail_flags = item['Tail flags'] == 'TRUE',
obj.tail_twitches = item['Tail twitches'] == 'TRUE',
obj.approaches = item['Approaches'] == 'TRUE',
obj.indifferent = item['Indifferent'] == 'TRUE',
obj.runs_from = item['Runs from'] == 'TRUE',
obj.save()
noduplicates.append(item['Unique Squirrel ID'])
def handle(self, *args, **kwargs):
path = abspath(kwargs['dataset_location'][0])
with open(path, 'r', encoding='utf-8') as file:
reader = csv.reader(file, delimiter=',')
dataset = list(reader)
num = len(dataset)
cur = 0
for row in dataset[1:]: # For ignoring the first line
cur = cur + 1
longitude = _format_longitude(row[0])
latitude = _format_latitude(row[1])
unique_squirrel_id = _format_unique_squirrel_id(row[2])
hectare = _format_hectare(row[3])
shift = _format_shift(row[4])
date = _format_date(row[5])
hectare_squirrel_number = _format_hectare_squirrel_number(
row[6])
age = _format_age(row[7])
primary_fur_color = _format_primary_fur_color(row[8])
highlight_fur_color = _format_highlight_fur_color(row[9])
combination_of_primary_and_highlight_color = _format_combination_of_primary_and_highlight_color(
row[10])
color_notes = _format_color_notes(row[11])
location = _format_location(row[12])
above_ground_sighter_measurement = _format_above_ground_sighter_measurement(
row[13])
specific_location = _format_specific_location(row[14])
running = _format_running(row[15])
chasing = _format_chasing(row[16])
climbing = _format_climbing(row[17])
eating = _format_eating(row[18])
foraging = _format_foraging(row[19])
other_activities = _format_other_activities(row[20])
kuks = _format_kuks(row[21])
quaas = _format_quaas(row[22])
moans = _format_moans(row[23])
tail_flags = _format_tail_flags(row[24])
tail_twitches = _format_tail_twitches(row[25])
approaches = _format_approaches(row[26])
indifferent = _format_indifferent(row[27])
runs_from = _format_runs_from(row[28])
other_interactions = _format_other_interactions(row[29])
lat_lng = _format_lat_lng(row[30])
zip_code = _format_zip_code(row[31])
community_districts = _format_community_districts(row[32])
borough_boundaries = _format_borough_boundaries(row[33])
city_council_districts = _format_city_council_districts(
row[34])
police_precincts = _format_police_precincts(row[35])
sighting = Sighting(
latitude=latitude,
longitude=longitude,
unique_squirrel_id=unique_squirrel_id,
hectare=hectare,
shift=shift,
date=date,
hectare_squirrel_number=hectare_squirrel_number,
age=age,
primary_fur_color=primary_fur_color,
highlight_fur_color=highlight_fur_color,
combination_of_primary_and_highlight_color=
combination_of_primary_and_highlight_color,
color_notes=color_notes,
location=location,
above_ground_sighter_measurement=
above_ground_sighter_measurement,
specific_location=specific_location,
running=running,
chasing=chasing,
climbing=climbing,
eating=eating,
foraging=foraging,
other_activities=other_activities,
kuks=kuks,
quaas=quaas,
moans=moans,
tail_flags=tail_flags,
tail_twitches=tail_twitches,
approaches=approaches,
indifferent=indifferent,
runs_from=runs_from,
other_interactions=other_interactions,
lat_lng=lat_lng,
zip_code=zip_code,
community_districts=community_districts,
borough_boundaries=borough_boundaries,
city_council_districts=city_council_districts,
police_precincts=police_precincts)
sighting.save()
print('%d of %d record(s) was inserted' % (cur, num))
def create_sighting(values):
# try:
s = Sighting()
s.sighting_id = int(values[0])
# Ugh... python hates times in 24:00 format... data contains it.
if " 24:" in values[1]:
values[1] = values[1].replace(" 24:", " 23:", 1)
dt = datetime.strptime(values[1], "%m/%d/%Y %H:%M")
dt = dt + timedelta(hours=1)
else:
dt = datetime.strptime(values[1], "%m/%d/%Y %H:%M")
s.occurred_at = pytz.utc.localize(dt)
s.city = values[2]
s.state = values[3]
s.country = values[4]
s.shape = values[5]
s.duration_seconds = float(values[6])
s.duration_text = values[7].strip()
s.description = values[8]
s.reported_on = datetime.strptime(values[9], '%m/%d/%Y').date()
s.latitude = float(values[10])
s.longitude = float(values[11])
# print(f"s is {s.__dict__}")
# if s.sighting_id == 4:
# try:
# s.save()
# except Exception as e:
# import ipdb; ipdb.set_trace()
# else:
s.save()
return True
