def load_csv2db(csv, header_map, resource, file_type='rep'): global done start = perf_counter() dataset = Dataset() pd_csv = csv csv_header = list(pd_csv) try: if file_type == 'rep': #concatentate the records data into a misc fields for later restoration pd_csv['misc'] = misc_col(csv, csv_header) #map replist headers to db headers pd_csv.rename(columns=header_map, inplace=True) #add id col for django import export pd_csv['Id'] = np.nan #import contact records dataset.csv = pd_csv.to_csv() resource.import_data(dataset, dry_run=False) except: print("lost the pandas csv") end = perf_counter() #stop timer time = end - start if file_type == 'rep': done = True uploadTime.objects.create(num_records=len(repContact.objects.all()), seconds=round(time, 2)) else: uploadTime.objects.create(num_records=len(sfcontact.objects.all()), seconds=round(time, 2)) return csv_header
def clean_events_sheet(self): uploaded_file = self.cleaned_data['events_sheet'] data = Dataset() if uploaded_file.name.endswith('.xlsx'): try: data.xlsx = uploaded_file.read() except Exception: logger.exception('Could not read Excel workbook upload') raise ValidationError('Could not read Excel workbook') elif uploaded_file.name.endswith('.csv'): try: data.csv = uploaded_file.read().decode('utf-8') except Exception: logger.exception('Could not read CSV upload') raise ValidationError('Could not read CSV') else: raise ValidationError('Unrecognized file type for "%s"' % uploaded_file.name) missing_field_list = ', '.join(self.required_fields - set(data.headers)) if missing_field_list: raise ValidationError( 'Missing fields in uploaded spreadsheet: %s' % missing_field_list) setattr(self, 'uploaded_events', data) return uploaded_file
def cmd_import(args): if not os.path.exists(COMO_BATTERY_FILE): current_dataset = create_database() else: current_dataset = read_database() if os.path.exists(args.get(0)): import_dataset = Dataset() with open(os.path.expanduser(args.get(0)), "r") as import_file: import_dataset.csv = import_file.read() import_dataset.dict = map(import_format, import_dataset.dict) new = current_dataset.stack(import_dataset).sort('time') with open(COMO_BATTERY_FILE, 'w') as como: como.write(zlib.compress(new.json)) puts(colored.white("battery statistics imported")) else: error("Couldn't open file: %s" % args.get(0))
def scrape(request): #apex callout sends json data with group, indi, and channel attrs if not filled then = 0 print(request.GET) logging.debug(request.GET) export = finra_check_job(request.GET) print('run scraper') print('get results from scraper') filename = 'filename="Scraper.csv"' response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = 'attachment; ' + filename dataset = Dataset() dataset.csv = export.to_csv(index=False) writer = csv.writer(response) for line in dataset: writer.writerow(line) return response
def parse(raw): data = Dataset() data.csv = raw print data.json
def import_csv(self, csv, uni, user): dataset = Dataset() dataset.csv = str(csv) return self.import_data(dataset, uni=uni, user=user)