def form_valid(self, form):
"""
Runs after form validation. Creates the ``Target``, and creates any ``TargetName`` or ``TargetExtra`` objects,
then runs the ``target_post_save`` hook and redirects to the success URL.
:param form: Form data for target creation
:type form: subclass of TargetCreateForm
"""
super().form_valid(form)
extra = TargetExtraFormset(self.request.POST)
names = TargetNamesFormset(self.request.POST)
if extra.is_valid() and names.is_valid():
extra.instance = self.object
extra.save()
names.instance = self.object
names.save()
else:
form.add_error(None, extra.errors)
form.add_error(None, extra.non_form_errors())
form.add_error(None, names.errors)
form.add_error(None, names.non_form_errors())
return super().form_invalid(form)
logger.info('Target post save hook: %s created: %s', self.object, True)
run_hook('target_post_save', target=self.object, created=True)
return redirect(self.get_success_url())
def create(self, request, *args, **kwargs):
request.data['data'] = request.FILES['file']
response = super().create(request, *args, **kwargs)
if response.status_code == status.HTTP_201_CREATED:
dp = DataProduct.objects.get(pk=response.data['id'])
try:
run_hook('data_product_post_upload', dp)
reduced_data = run_data_processor(dp)
if not settings.TARGET_PERMISSIONS_ONLY:
for group in response.data['group']:
assign_perm('tom_dataproducts.view_dataproduct', group,
dp)
assign_perm('tom_dataproducts.delete_dataproduct',
group, dp)
assign_perm('tom_dataproducts.view_reduceddatum',
group, reduced_data)
except Exception:
ReducedDatum.objects.filter(data_product=dp).delete()
dp.delete()
return Response(
{
'Data processing error':
'''There was an error in processing your DataProduct into \
individual ReducedDatum objects.'''
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return response
def post(self, request, *args, **kwargs):
"""
Method that handles POST requests for the ``DataProductSaveView``. Gets the observation facility that created
the data and saves the selected data products as ``DataProduct`` objects. Redirects to the
``ObservationDetailView`` for the specific ``ObservationRecord``.
:param request: Django POST request object
:type request: HttpRequest
"""
service_class = get_service_class(request.POST['facility'])
observation_record = ObservationRecord.objects.get(pk=kwargs['pk'])
products = request.POST.getlist('products')
if not products:
messages.warning(
request,
'No products were saved, please select at least one dataproduct'
)
elif products[0] == 'ALL':
products = service_class().save_data_products(observation_record)
messages.success(request, 'Saved all available data products')
else:
total_saved_products = []
for product in products:
saved_products = service_class().save_data_products(
observation_record, product)
total_saved_products += saved_products
run_hook('data_product_post_save', saved_products)
messages.success(
request, 'Successfully saved: {0}'.format('\n'.join(
[str(p) for p in saved_products])))
run_hook('multiple_data_products_post_save', total_saved_products)
return redirect(
reverse('tom_observations:detail',
kwargs={'pk': observation_record.id}))
def save(self, *args, **kwargs):
"""
Saves Target model data to the database, including extra fields. After saving to the database, also runs the
hook ``target_post_save``. The hook run is the one specified in ``settings.py``.
:Keyword Arguments:
* extras (`dict`): dictionary of key/value pairs representing target attributes
"""
extras = kwargs.pop('extras', {})
names = kwargs.pop('names', [])
created = False if self.id else True
super().save(*args, **kwargs)
if created:
for extra_field in settings.EXTRA_FIELDS:
if extra_field.get('default') is not None:
TargetExtra(target=self, key=extra_field['name'], value=extra_field.get('default')).save()
for k, v in extras.items():
target_extra, _ = TargetExtra.objects.get_or_create(target=self, key=k)
target_extra.value = v
target_extra.save()
for name in names:
name, _ = TargetName.objects.get_or_create(target=self, name=name)
name.save()
if not created:
run_hook('target_post_save', target=self, created=created)
def save(self, *args, **kwargs):
if self.id:
presave_data = ObservationRecord.objects.get(pk=self.id)
super().save(*args, **kwargs)
if self.status != presave_data.status:
run_hook('observation_change_state', self, presave_data.status)
else:
super().save(*args, **kwargs)
run_hook('observation_change_state', self, None)
def save(self, *args, **kwargs):
extras = kwargs.pop('extras', {})
created = False if self.id else True
super().save(*args, **kwargs)
for k, v in extras.items():
target_extra, _ = TargetExtra.objects.get_or_create(target=self,
key=k)
target_extra.value = v
target_extra.save()
run_hook('target_post_save', target=self, created=created)
def form_valid(self, form):
"""
Runs after ``DataProductUploadForm`` is validated. Saves each ``DataProduct`` and calls ``run_data_processor``
on each saved file. Redirects to the previous page.
"""
target = form.cleaned_data['target']
if not target:
observation_record = form.cleaned_data['observation_record']
target = observation_record.target
else:
observation_record = None
dp_type = form.cleaned_data['data_product_type']
data_product_files = self.request.FILES.getlist('files')
successful_uploads = []
for f in data_product_files:
dp = DataProduct(target=target,
observation_record=observation_record,
data=f,
product_id=None,
data_product_type=dp_type)
dp.save()
try:
run_hook('data_product_post_upload', dp)
reduced_data = run_data_processor(dp)
if not settings.TARGET_PERMISSIONS_ONLY:
for group in form.cleaned_data['groups']:
assign_perm('tom_dataproducts.view_dataproduct', group,
dp)
assign_perm('tom_dataproducts.delete_dataproduct',
group, dp)
assign_perm('tom_dataproducts.view_reduceddatum',
group, reduced_data)
successful_uploads.append(str(dp))
except InvalidFileFormatException as iffe:
ReducedDatum.objects.filter(data_product=dp).delete()
dp.delete()
messages.error(
self.request,
'File format invalid for file {0} -- error was {1}'.format(
str(dp), iffe))
except Exception:
ReducedDatum.objects.filter(data_product=dp).delete()
dp.delete()
messages.error(
self.request,
'There was a problem processing your file: {0}'.format(
str(dp)))
if successful_uploads:
messages.success(
self.request, 'Successfully uploaded: {0}'.format('\n'.join(
[p for p in successful_uploads])))
return redirect(form.cleaned_data.get('referrer', '/'))
def save(self, *args, **kwargs):
"""
Saves Target model data to the database, including extra fields. After saving to the database, also runs the
hook ``target_post_save``. The hook run is the one specified in ``settings.py``.
:Keyword Arguments:
* extras (`dict`): dictionary of key/value pairs representing target attributes
"""
extras = kwargs.pop('extras', {})
created = False if self.id else True
super().save(*args, **kwargs)
for k, v in extras.items():
target_extra, _ = TargetExtra.objects.get_or_create(target=self,
key=k)
target_extra.value = v
target_extra.save()
run_hook('target_post_save', target=self, created=created)
def form_valid(self, form):
target = form.cleaned_data['target']
observation_timestamp = form.cleaned_data.get('observation_timestamp',
None)
facility = form.cleaned_data.get('facility', None)
if not target:
observation_record = form.cleaned_data['observation_record']
target = observation_record.target
else:
observation_record = None
tag = form.cleaned_data['tag']
data_product_files = self.request.FILES.getlist('files')
successful_uploads = []
for f in data_product_files:
dp = DataProduct(target=target,
observation_record=observation_record,
data=f,
product_id=None,
tag=tag)
dp.save()
try:
run_hook('data_product_post_upload', dp, observation_timestamp,
facility)
successful_uploads.append(str(dp))
except InvalidFileFormatException:
ReducedDatum.objects.filter(data_product=dp).delete()
dp.delete()
messages.error(
self.request,
'There was a problem uploading your file--the file format was invalid for file: {0}'
.format(str(dp)))
except Exception:
ReducedDatum.objects.filter(data_product=dp).delete()
dp.delete()
messages.error(
self.request,
'There was a problem processing your file: {0}'.format(
str(dp)))
if successful_uploads:
messages.success(
self.request, 'Successfully uploaded: {0}'.format('\n'.join(
[p for p in successful_uploads])))
return redirect(form.cleaned_data.get('referrer', '/'))
def run(self):
failed_observations = [
obsr for obsr in
self.dynamic_cadence.observation_group.observation_records.all()
if obsr.failed
]
new_observations = []
for obs in failed_observations:
observation_payload = obs.parameters
facility = get_service_class(obs.facility)()
start_keyword, end_keyword = facility.get_start_end_keywords()
observation_payload = self.advance_window(
observation_payload,
start_keyword=start_keyword,
end_keyword=end_keyword)
obs_type = obs.parameters.get('observation_type', None)
form = facility.get_form(obs_type)(observation_payload)
form.is_valid()
observation_ids = facility.submit_observation(
form.observation_payload())
for observation_id in observation_ids:
# Create Observation record
record = ObservationRecord.objects.create(
target=obs.target,
facility=facility.name,
parameters=observation_payload,
observation_id=observation_id)
self.dynamic_cadence.observation_group.observation_records.add(
record)
self.dynamic_cadence.observation_group.save()
new_observations.append(record)
for obsr in new_observations:
### Sync with SNEx1
# Get the ID of the sequence in the SNEx1 obsrequests table
try:
snex_id = int(
self.dynamic_cadence.observation_group.name) #requestsid
except:
logger.info(
'Unable to find SNEx1 ID corresponding to observation group {}'
.format(self.dynamic_cadence.observation_group.name))
snex_id = ''
# Get the observation details from the submitted parameters
params = obsr.parameters
# Get the requestsgroup ID from the LCO API using the observation ID
obs_id = int(obsr.observation_id)
LCO_SETTINGS = settings.FACILITIES['LCO']
PORTAL_URL = LCO_SETTINGS['portal_url']
portal_headers = {
'Authorization': 'Token {0}'.format(LCO_SETTINGS['api_key'])
}
query_params = urlencode({'request_id': obs_id})
r = requests.get('{}/api/requestgroups?{}'.format(
PORTAL_URL, query_params),
headers=portal_headers)
requestgroups = r.json()
if requestgroups['count'] == 1:
requestgroup_id = int(requestgroups['results'][0]['id'])
# Use a hook to sync this observation request with SNEx1
run_hook('sync_observation_with_snex1', snex_id, params,
requestgroup_id)
return new_observations
def run(self):
# gets the most recent observation assuming the observation requests submitted from SNEx1 are being continuously sycned with SNEx2
last_obs = self.dynamic_cadence.observation_group.observation_records.order_by(
'-created').first()
# Make a call to the facility to get the current status of the observation
facility = get_service_class(last_obs.facility)()
facility.update_observation_status(
last_obs.observation_id) # Updates the DB record
last_obs.refresh_from_db() # Gets the record updates
# Boilerplate to get necessary properties for future calls
start_keyword, end_keyword = facility.get_start_end_keywords()
observation_payload = last_obs.parameters
# Cadence logic
# If the observation hasn't finished, do nothing
if not last_obs.terminal:
return
elif last_obs.failed: # If the observation failed
# Submit next observation to be taken as soon as possible with the same window length
window_length = parse(observation_payload[end_keyword]) - parse(
observation_payload[start_keyword])
observation_payload[start_keyword] = datetime.now().isoformat()
observation_payload[end_keyword] = (
parse(observation_payload[start_keyword]) +
window_length).isoformat()
else: # If the observation succeeded
# Advance window normally according to cadence parameters
observation_payload = self.advance_window(
observation_payload,
start_keyword=start_keyword,
end_keyword=end_keyword)
observation_payload = self.update_observation_payload(
observation_payload)
# Submission of the new observation to the facility
obs_type = last_obs.parameters.get('observation_type')
form = facility.get_form(obs_type)(observation_payload)
if form.is_valid():
observation_ids = facility.submit_observation(
form.observation_payload())
else:
logger.error(
msg=f'Unable to submit next cadenced observation: {form.errors}'
)
raise Exception(
f'Unable to submit next cadenced observation: {form.errors}')
# Creation of corresponding ObservationRecord objects for the observations
new_observations = []
for observation_id in observation_ids:
# Create Observation record
record = ObservationRecord.objects.create(
target=last_obs.target,
facility=facility.name,
parameters=observation_payload,
observation_id=observation_id)
# Add ObservationRecords to the DynamicCadence
self.dynamic_cadence.observation_group.observation_records.add(
record)
self.dynamic_cadence.observation_group.save()
new_observations.append(record)
# Update the status of the ObservationRecords in the DB
for obsr in new_observations:
facility = get_service_class(obsr.facility)()
facility.update_observation_status(obsr.observation_id)
### Sync with SNEx1
# Get the ID of the sequence in the SNEx1 obsrequests table
try:
snex_id = int(
self.dynamic_cadence.observation_group.name) #requestsid
except:
logger.info(
'Unable to find SNEx1 ID corresponding to observation group {}'
.format(self.dynamic_cadence.observation_group.name))
snex_id = ''
# Get the observation details from the submitted parameters
params = obsr.parameters
# Get the requestsgroup ID from the LCO API using the observation ID
obs_id = int(obsr.observation_id)
LCO_SETTINGS = settings.FACILITIES['LCO']
PORTAL_URL = LCO_SETTINGS['portal_url']
portal_headers = {
'Authorization': 'Token {0}'.format(LCO_SETTINGS['api_key'])
}
query_params = urlencode({'request_id': obs_id})
r = requests.get('{}/api/requestgroups?{}'.format(
PORTAL_URL, query_params),
headers=portal_headers)
requestgroups = r.json()
if requestgroups['count'] == 1:
requestgroup_id = int(requestgroups['results'][0]['id'])
# Use a hook to sync this observation request with SNEx1
run_hook('sync_observation_with_snex1', snex_id, params,
requestgroup_id)
return new_observations
def form_valid(self, form):
self.object = form.save(commit=True)
#logger.info('Target post save hook: %s created: %s', self.object, True)
run_hook('target_post_save', target=self.object, created=True)
return redirect(self.get_success_url())
def save(self, *args, **kwargs):
created = False if self.id else True
super().save(*args, **kwargs)
run_hook('target_post_save', target=self, created=created)
