def get_object_list(self, request): required_param = 'odk_form_id' odk_form_id = None try: odk_form_id = request.GET[required_param] except KeyError: pass try: document_id = request.GET['document_id'] odk_form_id = DocumentDetail.objects.get( document_id=document_id, doc_detail_type__name='odk_form_name').doc_detail_value except KeyError: pass if not odk_form_id: raise DatapointsException( '"{0}" is a required parameter for this request'.format( required_param)) try: odk_sync_object = OdkSync(odk_form_id, **{'user_id': request.user.id}) document_id_list, sync_result_data = odk_sync_object.main() except OdkJarFileException as e: raise DatapointsException(e.errorMessage) return Document.objects.filter(id__in=document_id_list).values()
def get_object_list(self, request): ''' ## when you upload a file, step one is getting data into source submission ## --> DocTransform <-- ## ## Step two is translating form source_submission into datapoints ## --> REfreshMaster <---- ## step three is aggregation ## agg refresh ## ''' try: doc_id = request.GET['document_id'] except KeyError: raise DatapointsException( message='Document_id is a required API param') # dt = DocTransform(request.user.id, doc_id) ran_complex_doc_transform = False try: dt = ComplexDocTransform(request.user.id, doc_id) dt.main() ran_complex_doc_transform = True except Exception as err: try: dt = DateDocTransform(request.user.id, doc_id) ssids = dt.process_file() except Exception as err: raise DatapointsException(message=err.message) mr = MasterRefresh(request.user.id, doc_id) mr.main() if ran_complex_doc_transform: doc_campaign_ids = set(list(DataPoint.objects\ .filter(source_submission__document_id = doc_id)\ .values_list('campaign_id',flat=True))) for c_id in doc_campaign_ids: ar = AggRefresh(c_id) # try/except block hack because tests fail otherwise try: with transaction.atomic(): ar.main() except TransactionManagementError as e: pass return Document.objects.filter(id=doc_id).values()
def get_object_list(self, request): try: doc_id = request.GET['document_id'] except KeyError: raise DatapointsException( message='Document_id is a required API param') # dt = DocTransform(request.user.id, doc_id) try: dt = SimpleDocTransform(request.user.id, doc_id) dt.main() except Exception as err: raise DatapointsException(message=err.message) return Document.objects.filter(id=doc_id).values()
def post_doc_data(self, post_data, user_id, doc_title, doc_id): # when posting from ODK, i dont add the file_meta.. from the webapp # i do. I should change so the post requests are consistent but # tryign to get this working for now. #TODO: better exception handling. This is kind of lame but handles the fact that test posts are different from #application posts. Need to investigate. if post_data == 'data:' or len(post_data) == 0: raise DatapointsException( message='file is empty please check the upload and try again') try: file_meta, base64data = post_data.split(',') except ValueError: base64data = post_data file_header = None file_content = None if '.csv' in doc_title: file_content = ContentFile(base64.b64decode(base64data)) file_header = file_content.readline() elif '.xlsx' in doc_title or '.xls' in doc_title: # workaround-- need to create the excel file in order to read from it new_file_path = settings.MEDIA_ROOT + doc_title new_file = open(new_file_path, 'wr') new_file.write(base64.b64decode(base64data)) new_file.close() the_file = open(new_file_path) try: file_df = read_excel(the_file) except Exception as err: os.remove(new_file_path) raise DatapointsException( message= 'There was an error with your file. Please check the upload and try again' ) file_content = ContentFile(file_df.to_csv()) file_header = file_content.readline() # delete the excel file os.remove(new_file_path) sd, created = Document.objects.update_or_create( id=doc_id, defaults={'doc_title': doc_title, 'created_by_id': user_id, \ 'file_header': file_header} ) sd.docfile.save(sd.guid, file_content) return sd
def obj_create(self, bundle, **kwargs): post_data = bundle.data try: campaign_id = int(post_data['id']) if campaign_id == -1: campaign_id = None except KeyError: campaign_id = None try: defaults = { 'name': str(post_data['name']), 'top_lvl_location_id': post_data['top_lvl_location_id'], 'top_lvl_indicator_tag_id': post_data['top_lvl_indicator_tag_id'], 'office_id': post_data['office_id'], 'campaign_type_id': post_data['campaign_type_id'], 'start_date': datetime.strptime(post_data['start_date'], '%Y-%m-%d'), 'end_date': datetime.strptime(post_data['end_date'], '%Y-%m-%d'), 'pct_complete': post_data['pct_complete'] } except Exception as error: print 'Please provide "{0}" for the campaign.'.format(error) raise DatapointsException( 'Please provide "{0}" for the campaign.'.format(error)) try: campaign, created = Campaign.objects.update_or_create( id=campaign_id, defaults=defaults) except Exception as error: raise DatapointsException(error) bundle.obj = campaign bundle.data['id'] = campaign.id return bundle
def get_object_list(self, request): document_id = request.GET['document_id'] try: dt = SimpleDocTransform(request.user.id, document_id) dt.main() except Exception as err: raise DatapointsException(message=err.message) return Document.objects.filter(id=document_id).values()
def get_object_list(self, request): try: doc_id = request.GET['document_id'] except KeyError: raise DatapointsException( message='Document_id is a required API param') # dt = DocTransform(request.user.id, doc_id) mr = MasterRefresh(request.user.id, doc_id) mr.main() doc_campaign_ids = set(list(DataPoint.objects\ .filter(source_submission__document_id = doc_id)\ .values_list('campaign_id',flat=True))) for c_id in doc_campaign_ids: ar = AggRefresh(c_id) return Document.objects.filter(id=doc_id).values()
def obj_create(self, bundle, **kwargs): post_data = bundle.data user_id = bundle.request.user.id try: dash_id = int(post_data['id']) except KeyError: dash_id = None title = post_data['title'] try: description = post_data['description'] except KeyError: description = '' try: layout = int(post_data['layout']) except KeyError: layout = 0 try: rows = json.loads(post_data['rows']) except KeyError: rows = None defaults = { 'id': dash_id, 'title': title, 'description': description, 'layout': layout, 'rows': rows } if (CustomDashboard.objects.filter(title=title).count() > 0 and (dash_id is None)): raise DatapointsException( 'the custom dashboard "{0}" already exists'.format(title)) dashboard, created = CustomDashboard.objects.update_or_create( id=dash_id, defaults=defaults) bundle.obj = dashboard bundle.data['id'] = dashboard.id return bundle
def file_to_source_submissions(self): #use a dictionary to make sure that there is a single value for each instance_guid. #duplicates are handled by overwriting old values batch = {} for submission in self.csv_df.itertuples(): ss, instance_guid = self.process_raw_source_submission(submission) if ss is not None and instance_guid is not None: ss['instance_guid'] = instance_guid batch[instance_guid] = ss object_list = [SourceSubmission(**v) for k, v in batch.iteritems()] try: ss = SourceSubmission.objects.bulk_create(object_list) except IntegrityError as e: raise DatapointsException(e.message) return
def file_to_source_submissions(self): batch = {} for submission in self.csv_df.itertuples(): ss, instance_guid = self.process_raw_source_submission(submission) if ss is not None and instance_guid is not None: ss['instance_guid'] = instance_guid batch[instance_guid] = ss object_list = [SourceSubmission(**v) for k, v in batch.iteritems()] try: ss = SourceSubmission.objects.bulk_create(object_list) except IntegrityError as e: raise DatapointsException(e.message) return
def __init__(self, user_id, document_id, raw_csv_df = None): self.user_id = user_id self.location_column, self.campaign_column, self.uq_id_column = \ ['geocode', 'campaign', 'unique_key'] self.date_column = 'data_date' self.document = Document.objects.get(id=document_id) self.file_path = str(self.document.docfile) if not isinstance(raw_csv_df, DataFrame): raw_csv_df = read_csv(settings.MEDIA_ROOT + self.file_path) csv_df = raw_csv_df.where((notnull(raw_csv_df)), None) ## if there is no uq id column -- make one ## if not self.uq_id_column in raw_csv_df.columns: try: csv_df[self.uq_id_column] = csv_df[self.location_column].map(str)+ csv_df[self.campaign_column] except Exception as err: if not self.date_column in csv_df.columns: dp_error_message = '%s is a required column.' %err.message raise DatapointsException(message=dp_error_message) self.csv_df = csv_df self.file_header = csv_df.columns self.meta_lookup = { 'location':{}, 'indicator':{}, 'campaign':{} } self.indicator_ids_to_exclude = Set([-1]) self.existing_submission_keys = SourceSubmission.objects.filter( document_id = self.document.id).values_list('instance_guid',flat=True) self.file_path = str(self.document.docfile)