def reportStatusToCakeApp(self):
try:
url = HPCConfig.getSpeciesReportURL(self.speciesId)
log.debug("url: %s", url)
values = {
'job_status': self.jobStatus,
'job_status_message': self.jobStatusMsg,
'dirty_occurrences': self.dirtyOccurrences
}
data = urllib.urlencode(values)
req = urllib2.Request(url, data)
connection = urllib2.urlopen(req)
responseContent = connection.read()
responseCode = connection.getcode()
if responseCode == 200:
log.debug("Reported job status, response: %s", responseContent)
return True
else:
log.warn("Failed to report job status, response: %s", responseContent)
return False
except (urllib2.URLError, urllib2.HTTPError, socket.timeout) as e:
log.warn("Error reporting job status: %s", e)
return False
def _writeCSVSpeciesJobFile(self):
try:
# Connect the DB
HPCConfig.connectDB()
try:
# Select the species row that matches this job's species
species_row = db.species.select()\
.where(db.species.c.id == self.speciesId)\
.execute().fetchone()
if species_row == None:
# We didn't find the species in the table..
# this shouldn't happen...
raise Exception("Couldn't find species with id " + self.speciesId + " in table. This shouldn't happen.")
else:
# We found it
# Now record the no. of dirtyOccurrences
dirtyOccurrences = species_row['num_dirty_occurrences']
self._setDirtyOccurrences(dirtyOccurrences)
self._setSpeciesCommonName(species_row['common_name'])
self._setSpeciesSciName(species_row['scientific_name'])
log.debug("Found %s dirtyOccurrences for species %s", dirtyOccurrences, self.speciesId)
# Create tempfiles to write our csv content to
priv_f = tempfile.NamedTemporaryFile(delete=False)
pub_f = tempfile.NamedTemporaryFile(delete=False)
metaData_f = tempfile.NamedTemporaryFile(delete=False)
try:
# Remember the path to the csv file
self._setPrivateTempfile(priv_f.name)
self._setPublicTempfile(pub_f.name)
self._setMetaDataTempfile(metaData_f.name)
log.debug("Writing public csv to: %s", pub_f.name)
log.debug("Writing private csv to: %s", priv_f.name)
log.debug("Writing meta data json to: %s", metaData_f.name)
# Write the metadata
# Get access to the sources for this species
# SELECT DISTINCT url, name, source_id FROM occurrences, sources WHERE occurrences.source_id=sources.id AND species_id=1;
source_rows = sqlalchemy.select(['url', 'name', 'source_id']).\
select_from(db.occurrences.join(db.sources)).\
where(db.occurrences.c.species_id == self.speciesId).\
distinct().\
execute()
meta_data_source_array = []
# Append to our meta data source array each source we found
for source_row in source_rows :
source_url = source_row['url']
source_name = source_row['name']
meta_data_source_array.append(
self.getMetaDataSourceDict(source_name, source_url)
)
# Dump the source metadata
metaDataString = json.dumps({
"harvester": {
"type": "directory",
"metadata": {
"occurrences": [{
"species_name" : self.getSpeciesNameForMetaData(),
"data_source_website" : meta_data_source_array
}],
"suitability": [{
"species_name" : self.getSpeciesNameForMetaData(),
"data_source_website" : meta_data_source_array
}]
}
}
})
metaData_f.write(metaDataString)
pub_writer = csv.writer(pub_f)
priv_writer = csv.writer(priv_f)
# Write the header
pub_writer.writerow(["LATDEC", "LONGDEC"])
priv_writer.writerow(["SPPCODE", "LATDEC", "LONGDEC"])
# Select the occurrences for this species
occurrence_rows = sqlalchemy.select([
'ST_X(location) as longitude',
'ST_Y(location) as latitude',
'ST_X(sensitive_location) as sensitive_longitude',
'ST_Y(sensitive_location) as sensitive_latitude']).\
select_from(db.occurrences.outerjoin(db.sensitive_occurrences)).\
where(db.occurrences.c.species_id == self.speciesId).\
where(or_(db.occurrences.c.classification == 'unknown', db.occurrences.c.classification >= 'core')).\
execute()
# Iterate over the occurrences, and write them to the csv
for occurrence_row in occurrence_rows:
pub_lat = occurrence_row['latitude']
pub_lng = occurrence_row['longitude']
pub_writer.writerow([pub_lat, pub_lng])
if occurrence_row['sensitive_longitude'] is None:
priv_lat = occurrence_row['latitude']
priv_lon = occurrence_row['longitude']
else:
priv_lat = occurrence_row['sensitive_latitude']
priv_lon = occurrence_row['sensitive_longitude']
priv_writer.writerow([self.speciesId, priv_lat, priv_lon])
finally:
# Be a good file citizen, and close the file handle
pub_f.close()
priv_f.close()
metaData_f.close()
finally:
# Dispose the DB
HPCConfig.disposeDB();
except Exception as e:
log.warn("Exception while trying to write CSV file species. Exception: %s", e)
raise
