def handle(self, **options):
organisation = csv.DictReader(open(os.path.join(DIR, 'Organisations.csv')))
for row in organisation:
o = Organisation(code=row['Code'], name=row['Name'])
o.save()
rfpt_org = Organisation.objects.get(code='RFPT')
sites = csv.DictReader(open(os.path.join(DIR, 'Sites.csv')))
for row in sites:
if not row['Latitude'].strip():
row['Latitude'] = None
if not row['Longitude'].strip():
row['Longitude'] = None
o = Site(code=row['Code'], latitude=row['Latitude'], longitude=row['Longitude'], description=row['Comments'], organisation=rfpt_org)
o.save()
recorders = csv.DictReader(open(os.path.join(DIR, 'Recorders.csv')))
for row in recorders:
o = Recorder(code=row['Code'], organisation=rfpt_org)
o.save()
deployments = csv.DictReader(open(os.path.join(DIR, 'Deployments.csv')))
for row in deployments:
site = Site.objects.get(code=row['Site'])
recorder = Recorder.objects.get(code=row['Recorder'].strip())
if not row['Deploy_time']:
row['Deploy_time'] = '00:00:00'
if not row['Recovery_time']:
row['Recovery_time'] = '00:00:00'
start = datetime.strptime(row['Deploy_date'] + ' ' + row['Deploy_time'], '%d/%m/%Y %H:%M:%S')
if row['Recovery_date'].strip():
end = datetime.strptime(row['Recovery_date'] + ' ' + row['Recovery_time'], '%d/%m/%Y %H:%M:%S')
else:
end = None
o = Deployment(site=site, recorder=recorder, start=start, end=end, comments=row['Comments'], owner=rfpt_org)
o.save()
def handle(self, **options):
organisation = csv.DictReader(
open(os.path.join(DIR, 'Organisations.csv')))
for row in organisation:
o = Organisation(code=row['Code'], name=row['Name'])
o.save()
vicu_org = Organisation.objects.get(code='VICU')
sites = csv.DictReader(open(os.path.join(DIR, 'Sites.csv')))
for row in sites:
if not row['Latitude'].strip():
row['Latitude'] = None
if not row['Longitude'].strip():
row['Longitude'] = None
o = Site(code=row['Code'],
latitude=row['Latitude'],
longitude=row['Longitude'],
description=row['Comments'],
organisation=vicu_org)
o.save()
recorders = csv.DictReader(open(os.path.join(DIR, 'Recorders.csv')))
for row in recorders:
o = Recorder(code=row['Code'], organisation=vicu_org)
o.save()
deployments = csv.DictReader(open(os.path.join(DIR,
'Deployments.csv')))
for row in deployments:
site = Site.objects.get(code=row['Site'])
recorder = Recorder.objects.get(code=row['Recorder'].strip())
#if not row['Deploy_time']:
# row['Deploy_time'] = '00:00:00'
#if not row['Recovery_time']:
# row['Recovery_time'] = '00:00:00'
#start = datetime.strptime(row['Deploy_date'] + ' ' + row['Deploy_time'], '%d/%m/%Y %H:%M:%S')
#if row['Recovery_date'].strip():
# end = datetime.strptime(row['Recovery_date'] + ' ' + row['Recovery_time'], '%d/%m/%Y %H:%M:%S')
#else:
# end = None
#used the following instead to try to remove 'Deploy_time' and 'Recovery_time' as we did not collect this information for our deployments.
start = pytz.utc.localize(
datetime.strptime(row['Start'], '%d/%m/%y'))
end = pytz.utc.localize(datetime.strptime(row['End'], '%d/%m/%y'))
o = Deployment(site=site,
recorder=recorder,
start=start,
end=end,
comments='None',
owner=vicu_org,
start_timezone='UTC')
o.save()
def handle(self, *args, **options):
logging.debug("started")
recordings = []
for root, dirs, files in os.walk(args[0]):
for f in files:
if f.endswith('.wav'):
recordings.append((f, os.path.join(root, f)))
recordings.sort()
#First make the organisation
try:
doc = Organisation.objects.get(code='doc')
except Organisation.DoesNotExist:
doc = Organisation(code='doc', name='Department of Conservation')
doc.save()
logging.info('Made a new organisation: doc')
#Now make the sites
site_codes = set(
['_'.join(f.split('_')[:2]) for (f, path) in recordings])
for code in site_codes:
try:
Site.objects.get(code=code, organisation=doc)
except Site.DoesNotExist:
Site(code=code, organisation=doc).save()
logging.info('Made a new site: %s' % code)
#Now make the deployments
tz = pytz.timezone('Etc/GMT-12')
old_site = ''
last_date = tz.localize(datetime.strptime('20000101', '%Y%m%d'))
count = 0
for f, path in recordings:
count += 1
site_code = '_'.join(f.split('_')[:2])
date = tz.localize(datetime.strptime(f.split('_')[2], '%y%m%d'))
starttime = tz.localize(
datetime.strptime('_'.join(f.split('_')[2:4]),
'%y%m%d_%H%M%S.wav'))
if site_code != old_site or (date - last_date).days > 1:
old_site = site_code
site = Site.objects.get(code=site_code, organisation=doc)
deployment, created = Deployment.objects.get_or_create(
site=site,
owner=doc,
start=date,
start_timezone='Pacific/Auckland')
if created:
deployment.save()
logging.info('Made a new deployment: %s, %s' %
(site, date))
last_date = date
if os.path.getsize(path) < MIN_FILE_SIZE:
logging.info('small file ignored: %s', path)
continue
md5 = get_md5(path)
try:
recording = Recording.objects.get(md5=md5)
logging.info('recording with same MD5 already in database: %s',
path)
recording.path = path
recording.save()
continue
except Recording.DoesNotExist:
logging.debug('recording not already in database: %s', path)
pass
try:
try:
Recording.objects.get(datetime=starttime,
deployment=deployment)
logging.error(
'recording already exists with the same startime (%s) and deployment (%s): %s',
starttime, deployment, path)
continue
except Recording.DoesNotExist:
pass
recording = Recording(datetime=starttime,
deployment=deployment,
path=path)
logging.debug('created the recording: %s', recording)
recording.save()
logging.debug('generate the snippets: %s', path)
#Now generate the snippets
if not recording.snippets.count():
try:
with closing(wave.open(path, 'r')) as w:
frames = w.getnframes()
rate = w.getframerate()
length = frames / float(rate)
snippet_length = 60
snippet_overlap = 0
snippet_minimum = 59.9
seconds = 0
while seconds + snippet_minimum < length:
offset = max(seconds - snippet_overlap, 0)
duration = min(
snippet_length + 2 * snippet_overlap,
length - offset)
Snippet(recording=recording,
offset=offset,
duration=duration).save()
seconds += snippet_length
except KeyboardInterrupt:
break
except:
logging.error('error extracting snippet: %s', path)
except Deployment.DoesNotExist:
logging.error('no matching deployment found: %s', path)
except Deployment.MultipleObjectsReturned:
logging.error('multiple matching deployment found: %s', path)
except IntegrityError:
logging.error('integrity error when trying to save file: %s',
path)
except wave.Error:
logging.error("doesn't seem to be a WAV file: %s", path)
except RecorderSiteError:
logging.error(
'unable to extract recorder or site from path: %s', path)
except KeyboardInterrupt:
break
def handle(self, *args, **options):
logging.debug("started")
recordings = []
for root, dirs, files in os.walk(args[0]):
for f in files:
if f.endswith('.wav'):
recordings.append((f, os.path.join(root, f)))
recordings.sort()
#First make the organisation
try:
doc = Organisation.objects.get(code='doc')
except Organisation.DoesNotExist:
doc = Organisation(code='doc', name='Department of Conservation')
doc.save()
logging.info('Made a new organisation: doc')
#Now make the sites
site_codes = set(['_'.join(f.split('_')[:2]) for (f, path) in recordings])
for code in site_codes:
try:
Site.objects.get(code=code, organisation=doc)
except Site.DoesNotExist:
Site(code=code, organisation=doc).save()
logging.info('Made a new site: %s' % code)
#Now make the deployments
tz = pytz.timezone('Etc/GMT-12')
old_site = ''
last_date = tz.localize(datetime.strptime('20000101', '%Y%m%d'))
count = 0
for f, path in recordings:
count += 1
site_code = '_'.join(f.split('_')[:2])
date = tz.localize(datetime.strptime(f.split('_')[2], '%y%m%d'))
starttime = tz.localize(datetime.strptime('_'.join(f.split('_')[2:4]), '%y%m%d_%H%M%S.wav'))
if site_code != old_site or (date - last_date).days > 1:
old_site = site_code
site = Site.objects.get(code=site_code, organisation=doc)
deployment, created = Deployment.objects.get_or_create(site=site, owner=doc,
start=date, start_timezone='Pacific/Auckland')
if created:
deployment.save()
logging.info('Made a new deployment: %s, %s' % (site, date))
last_date = date
if os.path.getsize(path) < MIN_FILE_SIZE:
logging.info('small file ignored: %s', path)
continue
md5 = get_md5(path)
try:
recording = Recording.objects.get(md5=md5)
logging.info('recording with same MD5 already in database: %s', path)
recording.path = path
recording.save()
continue
except Recording.DoesNotExist:
logging.debug('recording not already in database: %s', path)
pass
try:
try:
Recording.objects.get(datetime=starttime, deployment=deployment)
logging.error('recording already exists with the same startime (%s) and deployment (%s): %s',
starttime, deployment, path)
continue
except Recording.DoesNotExist:
pass
recording = Recording(datetime=starttime, deployment=deployment, path=path)
logging.debug('created the recording: %s', recording)
recording.save()
logging.debug('generate the snippets: %s', path)
#Now generate the snippets
if not recording.snippets.count():
try:
with closing(wave.open(path, 'r')) as w:
frames = w.getnframes()
rate = w.getframerate()
length = frames/float(rate)
snippet_length = 60
snippet_overlap = 0
snippet_minimum = 59.9
seconds = 0
while seconds + snippet_minimum < length:
offset = max(seconds - snippet_overlap, 0)
duration = min(snippet_length + 2*snippet_overlap, length - offset)
Snippet(recording=recording, offset=offset, duration=duration).save()
seconds += snippet_length
except KeyboardInterrupt:
break
except:
logging.error('error extracting snippet: %s', path)
except Deployment.DoesNotExist:
logging.error('no matching deployment found: %s', path)
except Deployment.MultipleObjectsReturned:
logging.error('multiple matching deployment found: %s', path)
except IntegrityError:
logging.error('integrity error when trying to save file: %s', path)
except wave.Error:
logging.error("doesn't seem to be a WAV file: %s", path)
except RecorderSiteError:
logging.error('unable to extract recorder or site from path: %s', path)
except KeyboardInterrupt:
break
