def handle(self, **options): organisation = csv.DictReader( open(os.path.join(DIR, 'Organisations.csv'))) for row in organisation: o = Organisation(code=row['Code'], name=row['Name']) o.save() vicu_org = Organisation.objects.get(code='VICU') sites = csv.DictReader(open(os.path.join(DIR, 'Sites.csv'))) for row in sites: if not row['Latitude'].strip(): row['Latitude'] = None if not row['Longitude'].strip(): row['Longitude'] = None o = Site(code=row['Code'], latitude=row['Latitude'], longitude=row['Longitude'], description=row['Comments'], organisation=vicu_org) o.save() recorders = csv.DictReader(open(os.path.join(DIR, 'Recorders.csv'))) for row in recorders: o = Recorder(code=row['Code'], organisation=vicu_org) o.save() deployments = csv.DictReader(open(os.path.join(DIR, 'Deployments.csv'))) for row in deployments: site = Site.objects.get(code=row['Site']) recorder = Recorder.objects.get(code=row['Recorder'].strip()) #if not row['Deploy_time']: # row['Deploy_time'] = '00:00:00' #if not row['Recovery_time']: # row['Recovery_time'] = '00:00:00' #start = datetime.strptime(row['Deploy_date'] + ' ' + row['Deploy_time'], '%d/%m/%Y %H:%M:%S') #if row['Recovery_date'].strip(): # end = datetime.strptime(row['Recovery_date'] + ' ' + row['Recovery_time'], '%d/%m/%Y %H:%M:%S') #else: # end = None #used the following instead to try to remove 'Deploy_time' and 'Recovery_time' as we did not collect this information for our deployments. start = pytz.utc.localize( datetime.strptime(row['Start'], '%d/%m/%y')) end = pytz.utc.localize(datetime.strptime(row['End'], '%d/%m/%y')) o = Deployment(site=site, recorder=recorder, start=start, end=end, comments='None', owner=vicu_org, start_timezone='UTC') o.save()
def handle(self, **options): organisation = csv.DictReader(open(os.path.join(DIR, 'Organisations.csv'))) for row in organisation: o = Organisation(code=row['Code'], name=row['Name']) o.save() rfpt_org = Organisation.objects.get(code='RFPT') sites = csv.DictReader(open(os.path.join(DIR, 'Sites.csv'))) for row in sites: if not row['Latitude'].strip(): row['Latitude'] = None if not row['Longitude'].strip(): row['Longitude'] = None o = Site(code=row['Code'], latitude=row['Latitude'], longitude=row['Longitude'], description=row['Comments'], organisation=rfpt_org) o.save() recorders = csv.DictReader(open(os.path.join(DIR, 'Recorders.csv'))) for row in recorders: o = Recorder(code=row['Code'], organisation=rfpt_org) o.save() deployments = csv.DictReader(open(os.path.join(DIR, 'Deployments.csv'))) for row in deployments: site = Site.objects.get(code=row['Site']) recorder = Recorder.objects.get(code=row['Recorder'].strip()) if not row['Deploy_time']: row['Deploy_time'] = '00:00:00' if not row['Recovery_time']: row['Recovery_time'] = '00:00:00' start = datetime.strptime(row['Deploy_date'] + ' ' + row['Deploy_time'], '%d/%m/%Y %H:%M:%S') if row['Recovery_date'].strip(): end = datetime.strptime(row['Recovery_date'] + ' ' + row['Recovery_time'], '%d/%m/%Y %H:%M:%S') else: end = None o = Deployment(site=site, recorder=recorder, start=start, end=end, comments=row['Comments'], owner=rfpt_org) o.save()
def handle(self, *args, **options): logging.debug("started") recordings = [] for root, dirs, files in os.walk(args[0]): for f in files: if f.endswith('.wav'): recordings.append((f, os.path.join(root, f))) recordings.sort() #First make the organisation try: doc = Organisation.objects.get(code='doc') except Organisation.DoesNotExist: doc = Organisation(code='doc', name='Department of Conservation') doc.save() logging.info('Made a new organisation: doc') #Now make the sites site_codes = set( ['_'.join(f.split('_')[:2]) for (f, path) in recordings]) for code in site_codes: try: Site.objects.get(code=code, organisation=doc) except Site.DoesNotExist: Site(code=code, organisation=doc).save() logging.info('Made a new site: %s' % code) #Now make the deployments tz = pytz.timezone('Etc/GMT-12') old_site = '' last_date = tz.localize(datetime.strptime('20000101', '%Y%m%d')) count = 0 for f, path in recordings: count += 1 site_code = '_'.join(f.split('_')[:2]) date = tz.localize(datetime.strptime(f.split('_')[2], '%y%m%d')) starttime = tz.localize( datetime.strptime('_'.join(f.split('_')[2:4]), '%y%m%d_%H%M%S.wav')) if site_code != old_site or (date - last_date).days > 1: old_site = site_code site = Site.objects.get(code=site_code, organisation=doc) deployment, created = Deployment.objects.get_or_create( site=site, owner=doc, start=date, start_timezone='Pacific/Auckland') if created: deployment.save() logging.info('Made a new deployment: %s, %s' % (site, date)) last_date = date if os.path.getsize(path) < MIN_FILE_SIZE: logging.info('small file ignored: %s', path) continue md5 = get_md5(path) try: recording = Recording.objects.get(md5=md5) logging.info('recording with same MD5 already in database: %s', path) recording.path = path recording.save() continue except Recording.DoesNotExist: logging.debug('recording not already in database: %s', path) pass try: try: Recording.objects.get(datetime=starttime, deployment=deployment) logging.error( 'recording already exists with the same startime (%s) and deployment (%s): %s', starttime, deployment, path) continue except Recording.DoesNotExist: pass recording = Recording(datetime=starttime, deployment=deployment, path=path) logging.debug('created the recording: %s', recording) recording.save() logging.debug('generate the snippets: %s', path) #Now generate the snippets if not recording.snippets.count(): try: with closing(wave.open(path, 'r')) as w: frames = w.getnframes() rate = w.getframerate() length = frames / float(rate) snippet_length = 60 snippet_overlap = 0 snippet_minimum = 59.9 seconds = 0 while seconds + snippet_minimum < length: offset = max(seconds - snippet_overlap, 0) duration = min( snippet_length + 2 * snippet_overlap, length - offset) Snippet(recording=recording, offset=offset, duration=duration).save() seconds += snippet_length except KeyboardInterrupt: break except: logging.error('error extracting snippet: %s', path) except Deployment.DoesNotExist: logging.error('no matching deployment found: %s', path) except Deployment.MultipleObjectsReturned: logging.error('multiple matching deployment found: %s', path) except IntegrityError: logging.error('integrity error when trying to save file: %s', path) except wave.Error: logging.error("doesn't seem to be a WAV file: %s", path) except RecorderSiteError: logging.error( 'unable to extract recorder or site from path: %s', path) except KeyboardInterrupt: break
def handle(self, *args, **options): logging.debug("started") recordings = [] for root, dirs, files in os.walk(args[0]): for f in files: if f.endswith('.wav'): recordings.append((f, os.path.join(root, f))) recordings.sort() #First make the organisation try: doc = Organisation.objects.get(code='doc') except Organisation.DoesNotExist: doc = Organisation(code='doc', name='Department of Conservation') doc.save() logging.info('Made a new organisation: doc') #Now make the sites site_codes = set(['_'.join(f.split('_')[:2]) for (f, path) in recordings]) for code in site_codes: try: Site.objects.get(code=code, organisation=doc) except Site.DoesNotExist: Site(code=code, organisation=doc).save() logging.info('Made a new site: %s' % code) #Now make the deployments tz = pytz.timezone('Etc/GMT-12') old_site = '' last_date = tz.localize(datetime.strptime('20000101', '%Y%m%d')) count = 0 for f, path in recordings: count += 1 site_code = '_'.join(f.split('_')[:2]) date = tz.localize(datetime.strptime(f.split('_')[2], '%y%m%d')) starttime = tz.localize(datetime.strptime('_'.join(f.split('_')[2:4]), '%y%m%d_%H%M%S.wav')) if site_code != old_site or (date - last_date).days > 1: old_site = site_code site = Site.objects.get(code=site_code, organisation=doc) deployment, created = Deployment.objects.get_or_create(site=site, owner=doc, start=date, start_timezone='Pacific/Auckland') if created: deployment.save() logging.info('Made a new deployment: %s, %s' % (site, date)) last_date = date if os.path.getsize(path) < MIN_FILE_SIZE: logging.info('small file ignored: %s', path) continue md5 = get_md5(path) try: recording = Recording.objects.get(md5=md5) logging.info('recording with same MD5 already in database: %s', path) recording.path = path recording.save() continue except Recording.DoesNotExist: logging.debug('recording not already in database: %s', path) pass try: try: Recording.objects.get(datetime=starttime, deployment=deployment) logging.error('recording already exists with the same startime (%s) and deployment (%s): %s', starttime, deployment, path) continue except Recording.DoesNotExist: pass recording = Recording(datetime=starttime, deployment=deployment, path=path) logging.debug('created the recording: %s', recording) recording.save() logging.debug('generate the snippets: %s', path) #Now generate the snippets if not recording.snippets.count(): try: with closing(wave.open(path, 'r')) as w: frames = w.getnframes() rate = w.getframerate() length = frames/float(rate) snippet_length = 60 snippet_overlap = 0 snippet_minimum = 59.9 seconds = 0 while seconds + snippet_minimum < length: offset = max(seconds - snippet_overlap, 0) duration = min(snippet_length + 2*snippet_overlap, length - offset) Snippet(recording=recording, offset=offset, duration=duration).save() seconds += snippet_length except KeyboardInterrupt: break except: logging.error('error extracting snippet: %s', path) except Deployment.DoesNotExist: logging.error('no matching deployment found: %s', path) except Deployment.MultipleObjectsReturned: logging.error('multiple matching deployment found: %s', path) except IntegrityError: logging.error('integrity error when trying to save file: %s', path) except wave.Error: logging.error("doesn't seem to be a WAV file: %s", path) except RecorderSiteError: logging.error('unable to extract recorder or site from path: %s', path) except KeyboardInterrupt: break