def clean_sourcefile(self):
"""
Make sure the sourcefile exists and that it's an xls file, and that the name indicates two streetnames.
"""
sourcefile_name = self.cleaned_data['sourcefile'].name
# if not os.path.isabs(sourcefile_name):
# raise forms.ValidationError("Sourcefile must be an absolute path to an excel file. Invalid value: %s" % sourcefile_name)
if sourcefile_name[-4:].lower() != ".xls" and sourcefile_name[-5:].lower() != ".xlsx":
raise forms.ValidationError("Sourcefile must be have a .xls|.xlsx suffix. Invalid value: %s" % sourcefile_name)
# set streetnames
self.cleaned_data['streetnames'] = CountsWorkbookParser.parseFilename(sourcefile_name)
if len(self.cleaned_data['streetnames']) not in [2,3]:
raise forms.ValidationError("Sourcefile name should be of the format streetname1_streetname2.xls or streetname_fromstreetname.tostreetname.xls. Invalid value: %s" % sourcefile_name)
return self.cleaned_data['sourcefile']
def clean_sourcefile(self):
"""
Make sure the sourcefile exists and that it's an xls file, and that the name indicates two streetnames.
"""
sourcefile_name = self.cleaned_data["sourcefile"].name
# if not os.path.isabs(sourcefile_name):
# raise forms.ValidationError("Sourcefile must be an absolute path to an excel file. Invalid value: %s" % sourcefile_name)
if sourcefile_name[-4:].lower() != ".xls" and sourcefile_name[-5:].lower() != ".xlsx":
raise forms.ValidationError(
"Sourcefile must be have a .xls|.xlsx suffix. Invalid value: %s" % sourcefile_name
)
# set streetnames
self.cleaned_data["streetnames"] = CountsWorkbookParser.parseFilename(sourcefile_name)
if len(self.cleaned_data["streetnames"]) not in [2, 3]:
raise forms.ValidationError(
"Sourcefile name should be of the format streetname1_streetname2.xls or streetname_fromstreetname.tostreetname.xls. Invalid value: %s"
% sourcefile_name
)
return self.cleaned_data["sourcefile"]
consolehandler = logging.StreamHandler()
consolehandler.setLevel(logging.DEBUG)
consolehandler.setFormatter(logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s'))
logger.addHandler(consolehandler)
debugFilename = "updateCountsWorkbooks.DEBUG.log"
debugloghandler = logging.StreamHandler(open(debugFilename, 'w'))
debugloghandler.setLevel(logging.DEBUG)
debugloghandler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M'))
logger.addHandler(debugloghandler)
files_to_process = sorted(os.listdir(DIR_TOPROCESS))
for file in files_to_process:
if file[-4:] !='.xls':
print "File suffix is not .xls: %s -- skipping" % file[-4:]
continue
logger.info("")
logger.info("Processing file %s" % file)
streetlist = CountsWorkbookParser.parseFilename(file)
# mainline
if len(streetlist) in [2,3]:
updateWorkbook(logger, DIR_TOPROCESS, DIR_OLDV10, DIR_NEWV11, file, "MAINLINE" if len(streetlist)==3 else "TURNS")
else:
logger.info(" Invalid workbook name %s" % file)
class UploadCountForm(forms.Form):
sourcefile = forms.FileField(max_length=150, help_text="Upload a count file to process.")
xl_parser = CountsWorkbookParser()
def clean_sourcefile(self):
"""
Make sure the sourcefile exists and that it's an xls file, and that the name indicates two streetnames.
"""
sourcefile_name = self.cleaned_data['sourcefile'].name
# if not os.path.isabs(sourcefile_name):
# raise forms.ValidationError("Sourcefile must be an absolute path to an excel file. Invalid value: %s" % sourcefile_name)
if sourcefile_name[-4:].lower() != ".xls" and sourcefile_name[-5:].lower() != ".xlsx":
raise forms.ValidationError("Sourcefile must be have a .xls|.xlsx suffix. Invalid value: %s" % sourcefile_name)
# set streetnames
self.cleaned_data['streetnames'] = CountsWorkbookParser.parseFilename(sourcefile_name)
if len(self.cleaned_data['streetnames']) not in [2,3]:
raise forms.ValidationError("Sourcefile name should be of the format streetname1_streetname2.xls or streetname_fromstreetname.tostreetname.xls. Invalid value: %s" % sourcefile_name)
return self.cleaned_data['sourcefile']
def read_sourcefile_and_insert_counts(self, request, file):
"""
Do the work! Read and insert the turn counts into the database.
Returns ( num_processed, error_string ), where num_processed will be -1 on error.
"""
# Figure out a filename
file_suffix_num = 1
new_filename = file.name
# check if the file already exists in uploads
while os.path.exists(os.path.join(settings.UPLOAD_DIR, new_filename)):
if file.name[-4:].lower() == ".xls":
new_filename = "%s_%d%s" % (file.name[:-4],file_suffix_num,file.name[-4:])
else:
new_filename = "%s_%d%s" % (file.name[:-5],file_suffix_num,file.name[-5:])
file_suffix_num += 1
# for now, save the file to c:\CountDracula\uploads
with open(os.path.join(settings.UPLOAD_DIR, new_filename), 'wb+') as destination:
for chunk in file.chunks():
destination.write(chunk)
# catch logs
buffer = StringIO.StringIO()
logHandler = logging.StreamHandler(buffer)
logHandler.setLevel(logging.INFO)
logging.getLogger().addHandler(logHandler)
logging.info("Saving into uploads as [%s]" % new_filename)
if len(self.cleaned_data['streetnames']) == 2:
# turn counts
processed = self.xl_parser.readAndInsertTurnCounts(os.path.join(settings.UPLOAD_DIR, new_filename),
self.cleaned_data['streetnames'][0],
self.cleaned_data['streetnames'][1],
request.user,
logging.getLogger())
else:
# mainline counts
processed = self.xl_parser.readAndInsertMainlineCounts(os.path.join(settings.UPLOAD_DIR, new_filename),
self.cleaned_data['streetnames'][0],
self.cleaned_data['streetnames'][1],
self.cleaned_data['streetnames'][2],
request.user,
logging.getLogger())
# stop catching logs
logging.getLogger().removeHandler(logHandler)
logHandler.flush()
buffer.flush()
return_str = buffer.getvalue()
# remove file on failure
if processed < 0:
os.remove(os.path.join(settings.UPLOAD_DIR, new_filename))
return_str += "Removed %s" % os.path.join(settings.UPLOAD_DIR,new_filename)
return_str = return_str.replace("<","<")
return_str = return_str.replace(">",">")
return_str = return_str.replace("\n","<br />")
return (processed, return_str)
consolehandler.setFormatter(logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s'))
logger.addHandler(consolehandler)
debugFilename = "insertSanFranciscoCounts_%s.DEBUG.log" % time.strftime("%Y%b%d.%H%M%S")
debugloghandler = logging.StreamHandler(open(debugFilename, 'w'))
debugloghandler.setLevel(logging.DEBUG)
debugloghandler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M'))
logger.addHandler(debugloghandler)
logger.info("Processing %s%s" % (COUNTS_INPUT, " starting with %s" % STARTFILE if STARTFILE else ""))
if FAIL_DIR:
logger.info("Putting files that CountDracula failed to process into %s" % FAIL_DIR)
if SUCCESS_DIR:
logger.info("Putting files that CountDracula successfully processed into %s" % SUCCESS_DIR)
xl_parser = CountsWorkbookParser()
# Read the counts. These reference the above streets and intersections.
mainline_processed_files = 0
mainline_processed_counts = 0
mainline_attempted_files = 0
turns_processed_files = 0
turns_processed_counts = 0
turns_attempted_files = 0
if os.path.isdir(COUNTS_INPUT):
dir = COUNTS_INPUT
files_to_process = sorted(os.listdir(COUNTS_INPUT))
else:
(dir,file) = os.path.split(COUNTS_INPUT)
files_to_process = [ file ]
consolehandler.setFormatter(
logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s'))
logger.addHandler(consolehandler)
debugFilename = "updateCountsWorkbooks.DEBUG.log"
debugloghandler = logging.StreamHandler(open(debugFilename, 'w'))
debugloghandler.setLevel(logging.DEBUG)
debugloghandler.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s %(message)s',
'%Y-%m-%d %H:%M'))
logger.addHandler(debugloghandler)
files_to_process = sorted(os.listdir(DIR_TOPROCESS))
for file in files_to_process:
if file[-4:] != '.xls':
print "File suffix is not .xls: %s -- skipping" % file[-4:]
continue
logger.info("")
logger.info("Processing file %s" % file)
streetlist = CountsWorkbookParser.parseFilename(file)
# mainline
if len(streetlist) in [2, 3]:
updateWorkbook(logger, DIR_TOPROCESS, DIR_OLDV10, DIR_NEWV11, file,
"MAINLINE" if len(streetlist) == 3 else "TURNS")
else:
logger.info(" Invalid workbook name %s" % file)
'%Y-%m-%d %H:%M'))
logger.addHandler(debugloghandler)
logger.info(
"Processing %s%s" %
(COUNTS_INPUT, " starting with %s" % STARTFILE if STARTFILE else ""))
if FAIL_DIR:
logger.info(
"Putting files that CountDracula failed to process into %s" %
FAIL_DIR)
if SUCCESS_DIR:
logger.info(
"Putting files that CountDracula successfully processed into %s" %
SUCCESS_DIR)
xl_parser = CountsWorkbookParser()
# Read the counts. These reference the above streets and intersections.
mainline_processed_files = 0
mainline_processed_counts = 0
mainline_attempted_files = 0
turns_processed_files = 0
turns_processed_counts = 0
turns_attempted_files = 0
if os.path.isdir(COUNTS_INPUT):
dir = COUNTS_INPUT
files_to_process = sorted(os.listdir(COUNTS_INPUT))
else:
(dir, file) = os.path.split(COUNTS_INPUT)
files_to_process = [file]