def clean_sourcefile(self): """ Make sure the sourcefile exists and that it's an xls file, and that the name indicates two streetnames. """ sourcefile_name = self.cleaned_data['sourcefile'].name # if not os.path.isabs(sourcefile_name): # raise forms.ValidationError("Sourcefile must be an absolute path to an excel file. Invalid value: %s" % sourcefile_name) if sourcefile_name[-4:].lower() != ".xls" and sourcefile_name[-5:].lower() != ".xlsx": raise forms.ValidationError("Sourcefile must be have a .xls|.xlsx suffix. Invalid value: %s" % sourcefile_name) # set streetnames self.cleaned_data['streetnames'] = CountsWorkbookParser.parseFilename(sourcefile_name) if len(self.cleaned_data['streetnames']) not in [2,3]: raise forms.ValidationError("Sourcefile name should be of the format streetname1_streetname2.xls or streetname_fromstreetname.tostreetname.xls. Invalid value: %s" % sourcefile_name) return self.cleaned_data['sourcefile']
def clean_sourcefile(self): """ Make sure the sourcefile exists and that it's an xls file, and that the name indicates two streetnames. """ sourcefile_name = self.cleaned_data["sourcefile"].name # if not os.path.isabs(sourcefile_name): # raise forms.ValidationError("Sourcefile must be an absolute path to an excel file. Invalid value: %s" % sourcefile_name) if sourcefile_name[-4:].lower() != ".xls" and sourcefile_name[-5:].lower() != ".xlsx": raise forms.ValidationError( "Sourcefile must be have a .xls|.xlsx suffix. Invalid value: %s" % sourcefile_name ) # set streetnames self.cleaned_data["streetnames"] = CountsWorkbookParser.parseFilename(sourcefile_name) if len(self.cleaned_data["streetnames"]) not in [2, 3]: raise forms.ValidationError( "Sourcefile name should be of the format streetname1_streetname2.xls or streetname_fromstreetname.tostreetname.xls. Invalid value: %s" % sourcefile_name ) return self.cleaned_data["sourcefile"]
consolehandler = logging.StreamHandler() consolehandler.setLevel(logging.DEBUG) consolehandler.setFormatter(logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')) logger.addHandler(consolehandler) debugFilename = "updateCountsWorkbooks.DEBUG.log" debugloghandler = logging.StreamHandler(open(debugFilename, 'w')) debugloghandler.setLevel(logging.DEBUG) debugloghandler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M')) logger.addHandler(debugloghandler) files_to_process = sorted(os.listdir(DIR_TOPROCESS)) for file in files_to_process: if file[-4:] !='.xls': print "File suffix is not .xls: %s -- skipping" % file[-4:] continue logger.info("") logger.info("Processing file %s" % file) streetlist = CountsWorkbookParser.parseFilename(file) # mainline if len(streetlist) in [2,3]: updateWorkbook(logger, DIR_TOPROCESS, DIR_OLDV10, DIR_NEWV11, file, "MAINLINE" if len(streetlist)==3 else "TURNS") else: logger.info(" Invalid workbook name %s" % file)
class UploadCountForm(forms.Form): sourcefile = forms.FileField(max_length=150, help_text="Upload a count file to process.") xl_parser = CountsWorkbookParser() def clean_sourcefile(self): """ Make sure the sourcefile exists and that it's an xls file, and that the name indicates two streetnames. """ sourcefile_name = self.cleaned_data['sourcefile'].name # if not os.path.isabs(sourcefile_name): # raise forms.ValidationError("Sourcefile must be an absolute path to an excel file. Invalid value: %s" % sourcefile_name) if sourcefile_name[-4:].lower() != ".xls" and sourcefile_name[-5:].lower() != ".xlsx": raise forms.ValidationError("Sourcefile must be have a .xls|.xlsx suffix. Invalid value: %s" % sourcefile_name) # set streetnames self.cleaned_data['streetnames'] = CountsWorkbookParser.parseFilename(sourcefile_name) if len(self.cleaned_data['streetnames']) not in [2,3]: raise forms.ValidationError("Sourcefile name should be of the format streetname1_streetname2.xls or streetname_fromstreetname.tostreetname.xls. Invalid value: %s" % sourcefile_name) return self.cleaned_data['sourcefile'] def read_sourcefile_and_insert_counts(self, request, file): """ Do the work! Read and insert the turn counts into the database. Returns ( num_processed, error_string ), where num_processed will be -1 on error. """ # Figure out a filename file_suffix_num = 1 new_filename = file.name # check if the file already exists in uploads while os.path.exists(os.path.join(settings.UPLOAD_DIR, new_filename)): if file.name[-4:].lower() == ".xls": new_filename = "%s_%d%s" % (file.name[:-4],file_suffix_num,file.name[-4:]) else: new_filename = "%s_%d%s" % (file.name[:-5],file_suffix_num,file.name[-5:]) file_suffix_num += 1 # for now, save the file to c:\CountDracula\uploads with open(os.path.join(settings.UPLOAD_DIR, new_filename), 'wb+') as destination: for chunk in file.chunks(): destination.write(chunk) # catch logs buffer = StringIO.StringIO() logHandler = logging.StreamHandler(buffer) logHandler.setLevel(logging.INFO) logging.getLogger().addHandler(logHandler) logging.info("Saving into uploads as [%s]" % new_filename) if len(self.cleaned_data['streetnames']) == 2: # turn counts processed = self.xl_parser.readAndInsertTurnCounts(os.path.join(settings.UPLOAD_DIR, new_filename), self.cleaned_data['streetnames'][0], self.cleaned_data['streetnames'][1], request.user, logging.getLogger()) else: # mainline counts processed = self.xl_parser.readAndInsertMainlineCounts(os.path.join(settings.UPLOAD_DIR, new_filename), self.cleaned_data['streetnames'][0], self.cleaned_data['streetnames'][1], self.cleaned_data['streetnames'][2], request.user, logging.getLogger()) # stop catching logs logging.getLogger().removeHandler(logHandler) logHandler.flush() buffer.flush() return_str = buffer.getvalue() # remove file on failure if processed < 0: os.remove(os.path.join(settings.UPLOAD_DIR, new_filename)) return_str += "Removed %s" % os.path.join(settings.UPLOAD_DIR,new_filename) return_str = return_str.replace("<","<") return_str = return_str.replace(">",">") return_str = return_str.replace("\n","<br />") return (processed, return_str)
consolehandler.setFormatter(logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')) logger.addHandler(consolehandler) debugFilename = "insertSanFranciscoCounts_%s.DEBUG.log" % time.strftime("%Y%b%d.%H%M%S") debugloghandler = logging.StreamHandler(open(debugFilename, 'w')) debugloghandler.setLevel(logging.DEBUG) debugloghandler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M')) logger.addHandler(debugloghandler) logger.info("Processing %s%s" % (COUNTS_INPUT, " starting with %s" % STARTFILE if STARTFILE else "")) if FAIL_DIR: logger.info("Putting files that CountDracula failed to process into %s" % FAIL_DIR) if SUCCESS_DIR: logger.info("Putting files that CountDracula successfully processed into %s" % SUCCESS_DIR) xl_parser = CountsWorkbookParser() # Read the counts. These reference the above streets and intersections. mainline_processed_files = 0 mainline_processed_counts = 0 mainline_attempted_files = 0 turns_processed_files = 0 turns_processed_counts = 0 turns_attempted_files = 0 if os.path.isdir(COUNTS_INPUT): dir = COUNTS_INPUT files_to_process = sorted(os.listdir(COUNTS_INPUT)) else: (dir,file) = os.path.split(COUNTS_INPUT) files_to_process = [ file ]
consolehandler.setFormatter( logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')) logger.addHandler(consolehandler) debugFilename = "updateCountsWorkbooks.DEBUG.log" debugloghandler = logging.StreamHandler(open(debugFilename, 'w')) debugloghandler.setLevel(logging.DEBUG) debugloghandler.setFormatter( logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M')) logger.addHandler(debugloghandler) files_to_process = sorted(os.listdir(DIR_TOPROCESS)) for file in files_to_process: if file[-4:] != '.xls': print "File suffix is not .xls: %s -- skipping" % file[-4:] continue logger.info("") logger.info("Processing file %s" % file) streetlist = CountsWorkbookParser.parseFilename(file) # mainline if len(streetlist) in [2, 3]: updateWorkbook(logger, DIR_TOPROCESS, DIR_OLDV10, DIR_NEWV11, file, "MAINLINE" if len(streetlist) == 3 else "TURNS") else: logger.info(" Invalid workbook name %s" % file)
'%Y-%m-%d %H:%M')) logger.addHandler(debugloghandler) logger.info( "Processing %s%s" % (COUNTS_INPUT, " starting with %s" % STARTFILE if STARTFILE else "")) if FAIL_DIR: logger.info( "Putting files that CountDracula failed to process into %s" % FAIL_DIR) if SUCCESS_DIR: logger.info( "Putting files that CountDracula successfully processed into %s" % SUCCESS_DIR) xl_parser = CountsWorkbookParser() # Read the counts. These reference the above streets and intersections. mainline_processed_files = 0 mainline_processed_counts = 0 mainline_attempted_files = 0 turns_processed_files = 0 turns_processed_counts = 0 turns_attempted_files = 0 if os.path.isdir(COUNTS_INPUT): dir = COUNTS_INPUT files_to_process = sorted(os.listdir(COUNTS_INPUT)) else: (dir, file) = os.path.split(COUNTS_INPUT) files_to_process = [file]