Exemple #1
0
    def clean_sourcefile(self):
        """
        Make sure the sourcefile exists and that it's an xls file, and that the name indicates two streetnames.
        """        
        sourcefile_name = self.cleaned_data['sourcefile'].name
        
        # if not os.path.isabs(sourcefile_name):
        #     raise forms.ValidationError("Sourcefile must be an absolute path to an excel file.  Invalid value: %s" % sourcefile_name)
        
       
        if sourcefile_name[-4:].lower() != ".xls" and sourcefile_name[-5:].lower() != ".xlsx":
            raise forms.ValidationError("Sourcefile must be have a .xls|.xlsx suffix.  Invalid value: %s" % sourcefile_name)
        
        # set streetnames
        self.cleaned_data['streetnames'] = CountsWorkbookParser.parseFilename(sourcefile_name)

        if len(self.cleaned_data['streetnames']) not in [2,3]:
            raise forms.ValidationError("Sourcefile name should be of the format streetname1_streetname2.xls or streetname_fromstreetname.tostreetname.xls.  Invalid value: %s" % sourcefile_name)
        
        return self.cleaned_data['sourcefile']
Exemple #2
0
    def clean_sourcefile(self):
        """
        Make sure the sourcefile exists and that it's an xls file, and that the name indicates two streetnames.
        """
        sourcefile_name = self.cleaned_data["sourcefile"].name

        # if not os.path.isabs(sourcefile_name):
        #     raise forms.ValidationError("Sourcefile must be an absolute path to an excel file.  Invalid value: %s" % sourcefile_name)

        if sourcefile_name[-4:].lower() != ".xls" and sourcefile_name[-5:].lower() != ".xlsx":
            raise forms.ValidationError(
                "Sourcefile must be have a .xls|.xlsx suffix.  Invalid value: %s" % sourcefile_name
            )

        # set streetnames
        self.cleaned_data["streetnames"] = CountsWorkbookParser.parseFilename(sourcefile_name)

        if len(self.cleaned_data["streetnames"]) not in [2, 3]:
            raise forms.ValidationError(
                "Sourcefile name should be of the format streetname1_streetname2.xls or streetname_fromstreetname.tostreetname.xls.  Invalid value: %s"
                % sourcefile_name
            )

        return self.cleaned_data["sourcefile"]
    
    consolehandler = logging.StreamHandler()
    consolehandler.setLevel(logging.DEBUG)
    consolehandler.setFormatter(logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s'))
    logger.addHandler(consolehandler)        
    
    debugFilename = "updateCountsWorkbooks.DEBUG.log"
    debugloghandler = logging.StreamHandler(open(debugFilename, 'w'))
    debugloghandler.setLevel(logging.DEBUG)
    debugloghandler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M'))
    logger.addHandler(debugloghandler)

    files_to_process = sorted(os.listdir(DIR_TOPROCESS))
    
    for file in files_to_process:
        if file[-4:] !='.xls':
            print "File suffix is not .xls: %s -- skipping" % file[-4:]
            continue
        
        logger.info("")
        logger.info("Processing file %s" % file)

        streetlist = CountsWorkbookParser.parseFilename(file)
        
        # mainline
        if len(streetlist) in [2,3]:
            updateWorkbook(logger, DIR_TOPROCESS, DIR_OLDV10, DIR_NEWV11, file, "MAINLINE" if len(streetlist)==3 else "TURNS")
        else:
            logger.info("  Invalid workbook name %s" % file)
            
Exemple #4
0
class UploadCountForm(forms.Form):

    sourcefile  = forms.FileField(max_length=150, help_text="Upload a count file to process.")
    xl_parser   = CountsWorkbookParser()
      
    def clean_sourcefile(self):
        """
        Make sure the sourcefile exists and that it's an xls file, and that the name indicates two streetnames.
        """        
        sourcefile_name = self.cleaned_data['sourcefile'].name
        
        # if not os.path.isabs(sourcefile_name):
        #     raise forms.ValidationError("Sourcefile must be an absolute path to an excel file.  Invalid value: %s" % sourcefile_name)
        
       
        if sourcefile_name[-4:].lower() != ".xls" and sourcefile_name[-5:].lower() != ".xlsx":
            raise forms.ValidationError("Sourcefile must be have a .xls|.xlsx suffix.  Invalid value: %s" % sourcefile_name)
        
        # set streetnames
        self.cleaned_data['streetnames'] = CountsWorkbookParser.parseFilename(sourcefile_name)

        if len(self.cleaned_data['streetnames']) not in [2,3]:
            raise forms.ValidationError("Sourcefile name should be of the format streetname1_streetname2.xls or streetname_fromstreetname.tostreetname.xls.  Invalid value: %s" % sourcefile_name)
        
        return self.cleaned_data['sourcefile']
        
    def read_sourcefile_and_insert_counts(self, request, file):
        """
        Do the work!  Read and insert the turn counts into the database.
        Returns ( num_processed, error_string ), where num_processed will be -1 on error.
        """
        # Figure out a filename
        file_suffix_num     = 1
        new_filename        = file.name
        # check if the file already exists in uploads
        while os.path.exists(os.path.join(settings.UPLOAD_DIR, new_filename)):
            if file.name[-4:].lower() == ".xls":
                new_filename = "%s_%d%s" % (file.name[:-4],file_suffix_num,file.name[-4:])
            else:
                new_filename = "%s_%d%s" % (file.name[:-5],file_suffix_num,file.name[-5:])
            file_suffix_num += 1
 
        
        # for now, save the file to c:\CountDracula\uploads
        with open(os.path.join(settings.UPLOAD_DIR, new_filename), 'wb+') as destination:
            for chunk in file.chunks():
                destination.write(chunk)
        
        # catch logs
        buffer      = StringIO.StringIO()
        logHandler  = logging.StreamHandler(buffer)
        logHandler.setLevel(logging.INFO)
        logging.getLogger().addHandler(logHandler)
        
        logging.info("Saving into uploads as [%s]" % new_filename)

        if len(self.cleaned_data['streetnames']) == 2:
            # turn counts
            processed = self.xl_parser.readAndInsertTurnCounts(os.path.join(settings.UPLOAD_DIR, new_filename), 
                                                               self.cleaned_data['streetnames'][0], 
                                                               self.cleaned_data['streetnames'][1], 
                                                               request.user,
                                                               logging.getLogger())
        else:
            # mainline counts
            processed = self.xl_parser.readAndInsertMainlineCounts(os.path.join(settings.UPLOAD_DIR, new_filename), 
                                                                   self.cleaned_data['streetnames'][0], 
                                                                   self.cleaned_data['streetnames'][1],
                                                                   self.cleaned_data['streetnames'][2],
                                                                   request.user, 
                                                                   logging.getLogger())  

        # stop catching logs
        logging.getLogger().removeHandler(logHandler)
        logHandler.flush()
        buffer.flush()
        return_str = buffer.getvalue()

        # remove file on failure
        if processed < 0:
            os.remove(os.path.join(settings.UPLOAD_DIR, new_filename))
            return_str += "Removed %s" % os.path.join(settings.UPLOAD_DIR,new_filename)

        return_str = return_str.replace("<","&lt;")
        return_str = return_str.replace(">","&gt;")
        return_str = return_str.replace("\n","<br />")
        return (processed, return_str)
    consolehandler.setFormatter(logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s'))
    logger.addHandler(consolehandler)        
    
    debugFilename = "insertSanFranciscoCounts_%s.DEBUG.log" % time.strftime("%Y%b%d.%H%M%S")
    debugloghandler = logging.StreamHandler(open(debugFilename, 'w'))
    debugloghandler.setLevel(logging.DEBUG)
    debugloghandler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M'))
    logger.addHandler(debugloghandler)

    logger.info("Processing %s%s" % (COUNTS_INPUT, " starting with %s" % STARTFILE if STARTFILE else ""))
    if FAIL_DIR:
        logger.info("Putting files that CountDracula failed to process into %s" % FAIL_DIR)
    if SUCCESS_DIR:
        logger.info("Putting files that CountDracula successfully processed into %s" % SUCCESS_DIR)
    
    xl_parser = CountsWorkbookParser()
   
    # Read the counts.  These reference the above streets and intersections.
    mainline_processed_files  = 0
    mainline_processed_counts = 0
    mainline_attempted_files  = 0
    turns_processed_files     = 0
    turns_processed_counts    = 0
    turns_attempted_files     = 0
    
    if os.path.isdir(COUNTS_INPUT):
        dir = COUNTS_INPUT
        files_to_process = sorted(os.listdir(COUNTS_INPUT))
    else:
        (dir,file) = os.path.split(COUNTS_INPUT)
        files_to_process = [ file ]
Exemple #6
0
    consolehandler.setFormatter(
        logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s'))
    logger.addHandler(consolehandler)

    debugFilename = "updateCountsWorkbooks.DEBUG.log"
    debugloghandler = logging.StreamHandler(open(debugFilename, 'w'))
    debugloghandler.setLevel(logging.DEBUG)
    debugloghandler.setFormatter(
        logging.Formatter('%(asctime)s %(levelname)s %(message)s',
                          '%Y-%m-%d %H:%M'))
    logger.addHandler(debugloghandler)

    files_to_process = sorted(os.listdir(DIR_TOPROCESS))

    for file in files_to_process:
        if file[-4:] != '.xls':
            print "File suffix is not .xls: %s -- skipping" % file[-4:]
            continue

        logger.info("")
        logger.info("Processing file %s" % file)

        streetlist = CountsWorkbookParser.parseFilename(file)

        # mainline
        if len(streetlist) in [2, 3]:
            updateWorkbook(logger, DIR_TOPROCESS, DIR_OLDV10, DIR_NEWV11, file,
                           "MAINLINE" if len(streetlist) == 3 else "TURNS")
        else:
            logger.info("  Invalid workbook name %s" % file)
Exemple #7
0
                          '%Y-%m-%d %H:%M'))
    logger.addHandler(debugloghandler)

    logger.info(
        "Processing %s%s" %
        (COUNTS_INPUT, " starting with %s" % STARTFILE if STARTFILE else ""))
    if FAIL_DIR:
        logger.info(
            "Putting files that CountDracula failed to process into %s" %
            FAIL_DIR)
    if SUCCESS_DIR:
        logger.info(
            "Putting files that CountDracula successfully processed into %s" %
            SUCCESS_DIR)

    xl_parser = CountsWorkbookParser()

    # Read the counts.  These reference the above streets and intersections.
    mainline_processed_files = 0
    mainline_processed_counts = 0
    mainline_attempted_files = 0
    turns_processed_files = 0
    turns_processed_counts = 0
    turns_attempted_files = 0

    if os.path.isdir(COUNTS_INPUT):
        dir = COUNTS_INPUT
        files_to_process = sorted(os.listdir(COUNTS_INPUT))
    else:
        (dir, file) = os.path.split(COUNTS_INPUT)
        files_to_process = [file]