示例#1
0
文件: postgres.py 项目: donbro/lsdb
def X_db_file_exists(cnx, in_dict): # , vol_id):
         
    gd = GetD(in_dict) 
    gd['vol_id'] = in_dict['vol_id'].encode('utf8')
    
    
    select_query = ( "select 1 from files "
            "where vol_id = %(vol_id)r and folder_id = %(folder_id)s "
            "and file_name = %(file_name)r and file_mod_date = %(file_mod_date)r "
            )


            # mysql.connector.errors.ProgrammingError: 1064 (42000): 
            # You have an error in your SQL syntax; check the manual that corresponds 
            #  to your MySQL server version for the right syntax to use 
            #  near ''vol0003' and folder_id = 2 and 
            #  file_name = 'TV Show\xe2\x80\x94single' and file' at line 1
            
            
    cursor = cnx.cursor()
    GPR.print_it(select_query % gd, 4)
    cursor.execute( select_query % gd )
    r = [z for z in cursor] 
    cursor.close()
    
    if r == [(1,)]:
        return True
    elif r == []:
        return False
    else:
        print "db_file_exists", r
        raise TypeError , ("db_file_exists says %r" % r)
示例#2
0
文件: postgres.py 项目: donbro/lsdb
    def test_051_postgres(self):
        """Is the server running?"""

        # test mode: either the server is running or it isn't.  both are ok as far as testing is concerned?

        # common error when no server is present:
        # note three addresses, all for "localhost"
        
        # could not connect to server: Connection refused
        #  Is the server running on host "localhost" (::1) and accepting
        #  TCP/IP connections on port 5432?
        # could not connect to server: Connection refused
        #  Is the server running on host "localhost" (127.0.0.1) and accepting
        #  TCP/IP connections on port 5432?
        # could not connect to server: Connection refused
        #  Is the server running on host "localhost" (fe80::1) and accepting
        #  TCP/IP connections on port 5432?
        

        try:
            cnx = psycopg2.connect(database='files',  user='******', host='localhost' )    
            GPR.print_attrs("connect", cnx, verbose_level_threshold=1)     # 4
        except psycopg2.OperationalError as err:
            errmsg = 'could not connect to server: Connection refused\n\tIs the server running on host "localhost" (::1) and accepting\n\tTCP/IP connections on port 5432?\ncould not connect to server: Connection refused\n\tIs the server running on host "localhost" (127.0.0.1) and accepting\n\tTCP/IP connections on port 5432?\ncould not connect to server: Connection refused\n\tIs the server running on host "localhost" (fe80::1) and accepting\n\tTCP/IP connections on port 5432?\n'
            self.assertEqual ( err.message , errmsg )
示例#3
0
文件: parse_args.py 项目: donbro/lsdb
    def test_010_do_parse_args(self):

        s = u'/Users/donb/Ashley+Roberts/'



        argv = []
        # argv = ["--help"]+[s]
        # argv = ["-rd 4"]
        argv = ["-d 2"]
        argv += ["-v"]   # how tohave "-v" mean one and -v2,  mean two?
        # argv += ["-v2"]
        # argv += ["-v"]
        # argv += ["-a"]
        argv += ["-p"]
        # argv += ["-f"] 
        argv += [s]
        


        (options, args) = do_parse_args(argv)

        GPR.print_dict("options (after optparsing)", options.__dict__, left_col_width=24, verbose_level_threshold=2)

        self.assertEqual(options, 
                Namespace(do_recursion=False, depth_limit=2, force_folder_scan=False, scan_hidden_files=False, scan_packages=True, verbose_level=2))
                
        self.assertEqual( args,  [u'/Users/donb/Ashley+Roberts/'] )
示例#4
0
文件: lsdb.py 项目: donbro/lsdb
 def do_push(in_value):
     """(push) (2 => 3) [(1, 40014149), (2, 42755279), (3, 45167012)]"""
     
     (depth, folder_file_id) = in_value
     # test for push of something other than just one.  shouldn't happen.
     if prev_depth != len(in_stak):
         GPR.print_it1( "(push) prev_depth (%d) != len(in_stak) (%d)\n" %  (prev_depth, len(in_stak)) )
     GPR.print_it0( "(push) (%r" % (prev_depth,) )
     in_stak.append(in_value)
     GPR.print_it0( "=> %r)" % (  depth) )
     GPR.print_it1( "%r"  %  in_stak                             )
     GPR.print_it1( "" )
     tally["(push)"] +=1
示例#5
0
文件: keystuff.py 项目: donbro/lsdb
def GetDR(item_dict, required_fields, quote_and_escape=True, verbose_level_threshold=3):
    """Convert from item_dict (Cocoa) forms to database-ready forms"""

    GPR.print_dict("GetDR(in)" , item_dict, 36, verbose_level_threshold)
   
    result_dict = {}
    for db_field_name in required_fields:
        
        if db_field_name in item_dict:
            dict_key_name = db_field_name            
        else:
            try:
                db_field_name_index = map( lambda y: y[0], databaseAndURLKeys).index(db_field_name)
                dict_key_name = databaseAndURLKeys[db_field_name_index][1]
            except ValueError:
                dict_key_name = db_field_name            

        # print  "%16s => %-36s :" % (db_field_name,  dict_key_name),
        
        #   do special processing based on database field name, not on inherent type of argument?
            
        if db_field_name in ['vol_id', 'file_name', 'file_uti']:
            c = item_dict[dict_key_name].encode('utf8')
            if quote_and_escape:
                result_dict[db_field_name] =  quote(escape(c))
            else:
                result_dict[db_field_name] =  c
                            
        elif db_field_name in ['file_create_date', 'file_mod_date']:
            d = item_dict[dict_key_name]
            if isinstance( d, str):
                c = d
            else:
                c = str(db_df.stringFromDate_(d))
            if quote_and_escape:
                result_dict[db_field_name] =  quote(escape(c))
            else:
                result_dict[db_field_name] =  c

        elif db_field_name in ['file_size', 'file_id', 'folder_id']: # 
            
            result_dict[db_field_name] =  int(item_dict[dict_key_name])

        else:
            result_dict[db_field_name] =  item_dict[dict_key_name]

        
    GPR.print_dict_no_repr("GetDR(out)" + ( "(q+e)" if quote_and_escape else "(no q+e)"), result_dict, 36, verbose_level_threshold)

    return result_dict
示例#6
0
文件: postgres.py 项目: donbro/lsdb
def db_select_vol_idz(cnx, d):
    
    gd = GetD(d) 
    
    select_query = ( "select  vol_id  from files "
                        "where  folder_id = 1 "
                        "and file_name = %(file_name)s and file_create_date = %(file_create_date)s "
                        )

    cursor = cnx.cursor()
    GPR.print_it(select_query % gd, 4)
    cursor.execute( select_query , gd )    
    r = [z for z in cursor] 
    if r == []:
        vol_id = None
        # print "vol_id is:", vol_id
    else:
        vol_id = r[0][0] # could have multiple results
    cursor.close()
    
    return  vol_id
示例#7
0
文件: lsdb.py 项目: donbro/lsdb
def vol_id_gen(cnx, in_gen):
    """processor for to find, store and then add vol_id to each item as it goes by"""
    
    # try:
    #   select vol_id from volumes_uuid (based on volume uuid)
    #   select vol_id from files (based on volume name and create date)
    #   do a "test" insert of the volume entry just to get the autocreated vol_id back. (then act like it didn't happen :)

    local_vol_id = None

    for in_dict in in_gen:

        if local_vol_id == None:        
            local_vol_id = db_get_vol_id(cnx, in_dict, local_vol_id)
            GPR.print_it2("vol_id_gen", "volume = %s, vol_id = %r" %  (in_dict[NSURLNameKey], local_vol_id), verbose_level_threshold=2)
            

        in_dict['vol_id'] = local_vol_id

        yield in_dict

    GPR.print_it("end vol_id_gen.", verbose_level_threshold=3)
示例#8
0
文件: postgres.py 项目: donbro/lsdb
def execute_update_query(cnx, update_sql, sql_dict, label='execute_update_query', verbose_level_threshold=3):

    cursor = cnx.cursor()

    s = sqlparse.format(update_sql % sql_dict, reindent=True, encoding='utf8')
    GPR.print_it2( label, s, verbose_level_threshold)
    
    try:
        cursor.execute( update_sql  %  sql_dict)
        cnx.commit()
    except mysql.connector.Error as err:
        if err.errno == 1062 and err.sqlstate == '23000':
            if True or GPR.verbose_level >= n:
                n1 = err.msg.index('Duplicate entry')
                n2 = err.msg.index('for key ')
                msg2 = err.msg[n1:n2-1]
                print "    "+repr(msg2)
        else:
            GPR.print_it2( label , "%r" % map(str , (err, err.errno , err.message , err.msg, err.sqlstate)), 0) # always print errors
    # Warning: It seems that you are trying to print a plain string containing unicode characters
    finally:
        cursor.close()
示例#9
0
文件: mysql_db.py 项目: donbro/lsdb
def db_connect():
    """open and return mysql connector object"""
    default_dict = {
        'user': '******',
        'password': '',
        'host': '127.0.0.1',
        'database': 'files',
        'buffered': True
    }

    config = ConfigParser.ConfigParser()
    config.readfp(open(CONFIG_FILE))
    config_dict = dict(config.items('dbstuff'))
    
    default_dict.update(config_dict)
    
    GPR.print_dict("db_connect(default_dict+config_dict)", default_dict, verbose_level_threshold=1) 
    
    # config.add_section('dbstuff')
    # for k,v in config_dict.items(): # self.__dict__.items():
    #     config.set('dbstuff', k, v)
    # with open(CONFIG_FILE, 'wb') as configfile:
    #     config.write(configfile)

    # GPR.print_dict("db_connect(default_dict)", default_dict, verbose_level_threshold=1) 

    try:
        cnx = mysql.connector.connect(**default_dict)
    except mysql.connector.Error as err:
        if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
            print("Username or password %r and %r?" % (default_dict['user'], default_dict['password']))
        elif err.errno == errorcode.ER_BAD_DB_ERROR:
            print "Database %r does not exist." % default_dict['database']
        else:
            print 'err:', err
            
    GPR.print_attrs("mysql.connector", cnx, verbose_level_threshold=4) 
    
    return cnx
示例#10
0
文件: postgres.py 项目: donbro/lsdb
def db_connect(verbose_level_threshold=2):
    """open and return mysql connector object"""

    default_dict = {
        'user': '******',
        'password': '',
        'host': '127.0.0.1',
        'database': 'files'
    }

    try:
        config = ConfigParser.ConfigParser()
        config.readfp(open(CONFIG_FILE))
        config_dict = dict(config.items('postgres'))
        default_dict.update(config_dict)
    except ConfigParser.NoSectionError as err:
        print err

    GPR.print_dict("postgres", default_dict, verbose_level_threshold=3) 

    try:
        cnx = psycopg2.connect(**default_dict) # connection could have a cursor factory?   
        GPR.print_attrs("connect", cnx, verbose_level_threshold=3)   
    except psycopg2.OperationalError as err:
        if err.message.startswith( 'could not connect to server: Connection refused') :
            print "Is the postgres server running?" 
        else:
            print err.message
            
        return
    
    return cnx




    
    # config.add_section('postgres')
    # for k,v in config_dict.items(): # self.__dict__.items():
    #     config.set('postgres', k, v)
    # with open(CONFIG_FILE, 'wb') as configfile:
    #     config.write(configfile)

    # GPR.print_dict("db_connect(default_dict)", default_dict, verbose_level_threshold=1) 

    try:
        cnx = mysql.connector.connect(**default_dict)
    except mysql.connector.Error as err:
        if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
            print("Username or password %r and %r?" % (default_dict['user'], default_dict['password']))
        elif err.errno == errorcode.ER_BAD_DB_ERROR:
            print "Database %r does not exist." % default_dict['database']
        else:
            print 'err:', err
            
    GPR.print_attrs("mysql.connector", cnx, verbose_level_threshold=4) 
    
    return cnx
示例#11
0
文件: postgres.py 项目: donbro/lsdb
def X_GetD(item_dict):
    """Convert from item_dict (Cocoa) forms to something that the database DBI can convert from"""

    d = {}
    for dk, fk in databaseAndURLKeys:
        if dk:
            if fk in [NSURLNameKey, NSURLTypeIdentifierKey]:
                d[dk] =  item_dict[fk].encode('utf8')
            elif dk in ['file_create_date', 'file_mod_date']:
                d[dk] =  str(item_dict[fk])[:-len(" +0000")] # "2011-07-02 21:02:54 +0000" => "2011-07-02 21:02:54"
            elif  type(item_dict[fk]) == objc._pythonify.OC_PythonLong:
                # print """"type(item_dict[fk]) = objc._pythonify.OC_PythonLong""", item_dict[fk], int(item_dict[fk]) 
                d[dk] = int(item_dict[fk])
            # elif 'objc._pythonify.OC_PythonLong' in str(type(item_dict[fk])):
            #     print """"nscfnumber" in str(type(item_dict[fk]):""", item_dict[fk], int(item_dict[fk]), objc._pythonify.OC_PythonLong
            #     d[dk] = int(item_dict[fk])
            else:
                # print type(item_dict[fk])                
                d[dk] =  item_dict[fk]

    GPR.print_dict("GetD result", d, 32, 4)

    return d
示例#12
0
文件: postgres.py 项目: donbro/lsdb
def db_insert_volume_data(cnx, vol_id, volume_url):
    """ insert/update volumes table with volume specific data, eg uuid, total capacity, available capacity """    

   #   get volume info

    values, error =  volume_url.resourceValuesForKeys_error_( ['NSURLVolumeUUIDStringKey',
                                                        'NSURLVolumeTotalCapacityKey',
                                                        'NSURLVolumeAvailableCapacityKey',
                                                        'NSURLVolumeSupportsPersistentIDsKey',
                                                        'NSURLVolumeSupportsVolumeSizesKey'] , None )
    if error is not None:
        print
        print error

    # volume_dict.update(dict(values))
    dv = dict(values)

    GPR.print_dict("volume info", values, 36, 1)
    
    # note: "on duplicate key update" of vol_total_capacity and vol_available_capacity.
    
    query = ("insert into volume_uuids "
                    "(vol_id, vol_uuid, vol_total_capacity, vol_available_capacity) "
                    "values ( %s, %s, %s, %s ) " 
                    "on duplicate key update "
                    "vol_total_capacity = values(vol_total_capacity), "
                    "vol_available_capacity = values(vol_available_capacity)"
                    )
    
    data = (vol_id, str(dv['NSURLVolumeUUIDStringKey']) ,
                    int(dv['NSURLVolumeTotalCapacityKey']),
                    int(dv['NSURLVolumeAvailableCapacityKey']) )
                    
    (l , vol_id) = execute_insert_query(cnx, query, data, 1)
    
    GPR.pr4(l, vol_id, "", data[1], 1)
示例#13
0
文件: postgres.py 项目: donbro/lsdb
def Z_db_query_folder(cnx,  item_dict):
    """get database contents of item as folder."""

    vol_id                  =       item_dict['vol_id'].encode('utf8')
    this_folder_id          =       item_dict['NSFileSystemFileNumber']
    
    
# in sql format "%r" is good because it provides quotes but will also insert u'vol0001'
# so either: str() the text (or utf8 encode it?) or format it like   '%s'  (but this is dumb quotes)

    sql = "select vol_id, folder_id, file_name, file_id, file_mod_date from files "+\
            "where vol_id = %r and folder_id = %d "
            
    data = (vol_id, this_folder_id )
    
    cur = cnx.cursor(cursor_class=MySQLCursorDict)

    GPR.print_it( sql % data, verbose_level_threshold=2)
    cur.execute( sql % data )
    cur.set_rel_name(in_rel_name="files_del") # need name at relation init time (ie, inside cursor fetch)
    r = cur.fetchall()
    cur.close()
    
    return r
示例#14
0
文件: mysql_db.py 项目: donbro/lsdb
def db_connect_psycopg2():
    
    cnx = psycopg2.connect(database='files',  user='******', host='localhost' )    
    GPR.print_attrs("psycopg2.connect", cnx, verbose_level_threshold=1)     # 4
    return cnx
示例#15
0
文件: lsdb.py 项目: donbro/lsdb
    def do_pop():
        """ (pop) (3 => 2)  [(1, 40014149), (2, 42755279)]"""

        # test for pop of something other than just one. can happen.  is okay:)
        if   depth+1 != len(in_stak):
            GPR.print_it1( " (pop) depth+1 (%d) != len(in_stak) (%d)\n" %  (depth+1, len(in_stak)) )

        GPR.print_it0( " (pop) (%r"% (len(in_stak), ) )
        tally["(pop)"] +=1

        (d,ffid) = in_stak.pop()
        GPR.print_it0( "=> %r) "% (len(in_stak), ) )
        
        # this should really be a method on a subclass of stak?
        
        if hasattr(in_stak, 'RS'):
            if (d,ffid) in in_stak.RS: # in_stak.RS.keys():
                GPR.print_it0( "key %r in stak: %r" % ((d,ffid), in_stak) )
                GPR.print_it1( "(dict)")
                GPR.print_it1("")
                return in_stak.RS.pop((d,ffid))   # in_stak.RS[(d,ffid)]
            else:
                # not error, directory was up to date(?)
                GPR.print_it1( "key %r not in stak: %r" % ((d,ffid), in_stak) ) 
                GPR.print_it1("")
                return []
        else:
            GPR.print_it0( "in_stak has no attr 'RS'" )
            GPR.print_it1( "%r"  %  in_stak )
            GPR.print_it1("")
            return []
示例#16
0
文件: lsdb.py 项目: donbro/lsdb
def main():

    #   some favorite testing files


    s = '/Volumes/Ulysses/bittorrent'
    s =     u'/Users/donb/Downloads/incomplete'
    s = '/Volumes/Ulysses/TV Shows/Nikita/'

    # package
    s = u"/Users/donb/Documents/Installing Evernote v. 4.6.2—Windows Seven.rtfd"


    s = '/Volumes/Ulysses/bittorrent'


    s = u'/Users/donb/Downloads/incomplete'


    s = u'/Users/donb/Ashley+Roberts/'
    
    

    
    
    s = "/Volumes/Taos/videogame/"
    
    
    s = "/Volumes/Daytona/TV Series/Americas Next Top Model"

    s = '.'
    
    s = "/Volumes/Taos/videogame/Perfect Dark/Joanna Dark/"

    s = u'/Users/donb/Ashley+Roberts/'
    s = u"~/Catherine Video Review.mp4"
    s = "/Volumes/Brandywine/TV Show—single/"
    
    s = '/Volumes/Ulysses/TV Shows/Nikita/'
    s = '/Volumes/Ulysses/bittorrent/'
    s = "/Volumes/Corinna"
    s = "/Volumes/Corinna/Actress/Alison Armitage"
    s = "/Volumes/Corinna/Actress/Keira Knightley/Keira Knightley - Kenneth Willardt's GQ Photoshoot"
    
    s = "/Volumes/Dunharrow"

    # hack to have Textmate run with hardwired arguments while command line can be free…
    if os.getenv('TM_LINE_NUMBER' ):
        argv = []

        argv += ["-v"] # verbose_level = 2
        argv += ["-v"]
#         argv += ["-v"]  # verbose_level = 4
        # argv = ["-d 3"]        
        argv += ["-f"]          # force folder scan
        # argv += ["-p"]      # scanning packages
        argv += [s]
    else:
        argv = sys.argv[1:]
    

    (options, args) = do_parse_args(argv)
    
    # no args means do the current directory
    
    if args == []: 
        args = ["."]
    
    args2 = []
    for a in args:
        try:
            unicode(a)
        except UnicodeDecodeError:
            a2 = a.decode('utf8')
            # print "arg [  %s  ] is a unicode string" % (a2, )
            GPR.print_it2("arg is a unicode string", a2, verbose_level_threshold=1)
            args2.append(a2)
        else:
            args2.append(a)
    args = args2
        
    args = [os.path.abspath(os.path.expanduser(a)) for a in args]
    
    GPR.verbose_level = options.verbose_level

    GPR.print_list("sys.argv", sys.argv, verbose_level_threshold=3)

    # display list of timezones
    if options.verbose_level >= 4:
        print_timezones("time_zones")

    GPR.print_dict("options (after optparsing)", options.__dict__, left_col_width=24, verbose_level_threshold=2)

    GPR.print_list("args (after optparsing)", args, verbose_level_threshold=3)
        
    do_args(args, options)
示例#17
0
文件: lsdb.py 项目: donbro/lsdb
def do_args(args, options):
    """do_args is the high-level, self-contained routine most like the command-line invocation"""

    cnx = db_connect()
    
    required_fields =  ['vol_id', 'folder_id', 'file_name', 'file_id', 'file_size', 'file_create_date', 'file_mod_date', 'file_uti' ]

    try:
        for basepath in args:
            
            for arg_dict in do_arg_gen(basepath, cnx, options):  

                # depth 0 should be a fully-realized level
                if (arg_dict['depth'] < 0):
                    if not ('directory_is_up_to_date' in arg_dict) or not arg_dict['directory_is_up_to_date']:
                        sql_dict = GetDR(arg_dict, required_fields)
                        sql_dict['file_mod_date'] = "'1970-01-01 00:00:00'" # args are escaped and quoted at this point
                        add_file_sql = ("insert into files "
                                        "(vol_id, folder_id, file_name, file_id, file_size, file_create_date, file_mod_date, file_uti) "
                                        "values "
                                        "( %(vol_id)s, %(folder_id)s, %(file_name)s, %(file_id)s, %(file_size)s, %(file_create_date)s, "
                                        "%(file_mod_date)s, %(file_uti)s ) "
                                        )
                        # execute_update_query(cnx, add_file_sql , sql_dict, label='(depth < 0)', verbose_level_threshold=3 )

                        db_execute_sql(cnx, add_file_sql % sql_dict, label='(depth < 0)', verbose_level_threshold=2)

                    GPR.pr7z( arg_dict ) 

                elif 'sql_action' in arg_dict:

                    if arg_dict['sql_action'] in  ["update_directory", "insert"]:
                        
                        # technically, we are updating (ie, completing) the directory
                        #  before we do the directory entries?  consistency problem if we fail?

                        add_file_sql = ("insert into files "
                                        "(vol_id, folder_id, file_name, file_id, file_size, file_create_date, file_mod_date, file_uti) "
                                        "values "
                                        "( %(vol_id)s, %(folder_id)s, %(file_name)s, %(file_id)s, %(file_size)s, %(file_create_date)s, "
                                        "%(file_mod_date)s, %(file_uti)s ) "
                                        )

                        # execute_update_query(cnx, add_file_sql , sql_dict, label=arg_dict['sql_action'], verbose_level_threshold=2)  # sql and dict are "%"'ed inside function
                        
                        db_execute(cnx, add_file_sql, arg_dict, required_fields, label="do_args" + arg_dict['sql_action'], verbose_level_threshold=2)

                    else:

                        sql_dict = GetDR(arg_dict, required_fields)
                        GPR.print_it(add_file_sql % sql_dict, 3)
                                                            
                    GPR.pr7z( arg_dict ) 
                elif (arg_dict['depth'] == 0):

                    GPR.pr7z( arg_dict , verbose_level_threshold=1) 
                        
                else:
                    
                    GPR.pr7z( arg_dict , verbose_level_threshold=2) 
                
                    
            

    except MyError, err:
        print err.description
示例#18
0
文件: lsdb.py 项目: donbro/lsdb
def do_arg_gen(basepath, cnx, options):  
    
    x_gen = files_generator(basepath, options)

    my_stak = stak() # contains self.RS

    fs_gen = partial(files_stak_gen, in_stak=my_stak, cnx=cnx) 

    y_gen = partial(vol_id_gen, cnx)

    # for fs_dict in  y_gen( fs_gen( x_gen ) ):    
    for fs_dict in  fs_gen( y_gen( x_gen ) ):    
    
        depth = fs_dict['depth']
        file_id = fs_dict['NSFileSystemFileNumber']

        if is_a_directory(fs_dict, options):
            GPR.print_it0( "(directory)" )
            if (depth < 0 ):
                GPR.print_it1( "(depth < 0)" )  # eol
            else:
                dir_is_up_to_date = not options.force_folder_scan and db_file_exists(cnx, fs_dict) 
                fs_dict['directory_is_up_to_date'] = dir_is_up_to_date                  
                if (dir_is_up_to_date):
                    GPR.print_it1( "(up_to_date)" )  # eol
                else:
                    GPR.print_it0( "(to_be_scanned)" )
                    GPR.print_it0( "(scanning)" )
                    rel = db_query_folder(cnx, fs_dict)
                    GPR.print_it0( "(len=%r)" % (len(rel),) )
                    GPR.print_it0( "(storing at) %r" % ( (depth+1, file_id) ,) )
                    # don't store those of zero length because you'll never pop them?  no:
                    #   do store zero lengths because we will want to "subtract against them" in directory check.
                    my_stak.RS[ (depth+1, file_id)  ] =  rel
                    GPR.print_it1( "stak: %r" % (my_stak,) ) # eol
                    fs_dict['sql_action'] = "update_directory"
                    GPR.print_it0( "(update directory)" )
                    

        folder_id = fs_dict['NSFileSystemFolderNumber']

        fs_dict['current_item_directory_is_being_checked'] =  (depth, folder_id) in my_stak.RS
        if fs_dict['current_item_directory_is_being_checked']:          
            GPR.print_it0( "(check against container directory %r)" % ((depth,folder_id ) ,) , verbose_level_threshold=3)
            
            vol_id          = fs_dict['vol_id']
            filename        = fs_dict[NSURLNameKey]                                 # still unicode, encode late.

            required_fields =  ['vol_id', 'folder_id', 'file_name', 'file_id', 'file_mod_date' ]

            sql_dict = GetDR(fs_dict, required_fields, quote_and_escape=False, verbose_level_threshold=4)  #ie, not for sql, just for values

            rs = (  vol_id,   folder_id,  filename,  file_id, sql_dict['file_mod_date'])
                        
            rs2 = tuple([filename if k == 'file_name' else sql_dict[k] for k in required_fields]) # unicode filename
            
            if rs != rs2:
                print "\n%r != %r\n" % (rs, rs2)

            if rs in my_stak.RS[ (depth,folder_id ) ]:
                my_stak.RS[ (depth,folder_id ) ] -= rs   
                GPR.print_it0( "(ignore)" )
                GPR.print_it0( "(already in database) %r" % my_stak )
            else:
                fs_dict['to_be_inserted'] = True
                fs_dict['sql_action'] = "insert"
                GPR.print_it0( "(insert)" )
                
            #     
            # try:                
            #     my_stak.RS[ (depth,folder_id ) ] -= rs       # my_stak.RS[ (depth,folder_id ) ]._convert_to_row(rs) , or just, tuple(in_row) 
            #     GPR.print_it0( "(ignore)" )
            #     GPR.print_it0( "(already in database) %r" % my_stak )
            # except KeyError:  # ie, the key of rs is not in the set of stak.ts[(d,fid)]
            #     fs_dict['to_be_inserted'] = True
            #     fs_dict['sql_action'] = "insert"
            #     GPR.print_it0( "(insert)" )

        yield fs_dict

    #   end gen

    if options.verbose_level >= 3:
        print "end do_arg_gen. my_stak is", my_stak
示例#19
0
文件: lsdb.py 项目: donbro/lsdb
def files_generator(basepath, options):
    """a generator which yields all files (as file_dicts) including volume, superfolder(s), 
            basepath, and then all subfiles (subject to depth_limit and enumerator options). """

    GPR.print_it25("files_generator", basepath, 2)

    superfolders_list = []
    
    basepath_url =  NSURL.fileURLWithPath_(basepath)
    
    # begin loop going upwards
    url =  NSURL.fileURLWithPath_(basepath)
    while True:
        d1 = GetURLValues(url, enumeratorURLKeys)
        superfolders_list.insert(0,d1)
        if d1[NSURLIsVolumeKey]: 
            break
        # go "upwards" one level (towards volume)
        url = url.URLByDeletingLastPathComponent()              

    GPR.print_superfolders_list("volume, superfolder(s)", superfolders_list, 4)

    # now go back down, yielding dict objects at each step:
    n = len(superfolders_list)
    for i, superfolder_dict in enumerate(superfolders_list):  
        superfolder_dict['depth'] = i+1-n
        yield superfolder_dict 

    # last dict in superfolder list is the basepath_dict
    basepath_dict =  superfolder_dict                          

    item_is_package = is_item_a_package(basepath_url)    
    if basepath_dict[NSURLIsDirectoryKey] and item_is_package and not options.scan_packages:
        GPR.print_it("\nbasepath is a directory and a package but we're not scanning packages.\n", 1)
        return

    # we've yielded basepath above, don't enumerate if basepath is not a directory (or package and we want packages)
    
    if basepath_dict[NSURLIsDirectoryKey]:
    
        enumeratorOptionKeys = 0L
        if not options.scan_packages:
            enumeratorOptionKeys |= NSDirectoryEnumerationSkipsPackageDescendants
        if not options.scan_hidden_files:
            enumeratorOptionKeys |= NSDirectoryEnumerationSkipsHiddenFiles

        enumerator = sharedFM.enumeratorAtURL_includingPropertiesForKeys_options_errorHandler_(
                                                                                        basepath_url,   
                                                                                        enumeratorURLKeys, 
                                                                                        enumeratorOptionKeys, 
                                                                                        error_handler_for_enumerator )                                        
        for url in enumerator:
            item_dict = GetURLValues(url, enumeratorURLKeys)
            depth = enumerator.level()                
            item_dict['depth'] = depth

            if options.depth_limit and (depth >= options.depth_limit-1):
                enumerator.skipDescendants()
        
            yield item_dict

    GPR.print_it2("end files_generator", basepath, verbose_level_threshold=3)
示例#20
0
文件: lsdb.py 项目: donbro/lsdb
def db_execute_relation(cnx, sql_query, item_dict, required_fields, label, verbose_level_threshold=2):
    
    """db_execute is GetDR, sqlparse, cursor.execute, return rel.add( r ) for r in cursor and cursor.close """
    
    sql_dict = GetDR(item_dict, required_fields, verbose_level_threshold=verbose_level_threshold)    
    sql_query = sql_query % sql_dict    
    
    s = sqlparse.format( sql_query, reindent=True, encoding='utf8')
    GPR.print_it2( label, s, verbose_level_threshold) # 4

    try:
        cursor = cnx.cursor()
        cursor.execute( sql_query )    
        column_names = [column.name for column in cursor.description]
        rel = relation( column_names, [] , "pg_del_files")
        for r in cursor:
            l = list(r)
            l[2] = l[2].decode('utf8') # file_name
            l[4] = str(l[4])
            rel.add( l ) 
        cnx.commit()
        
        # r = [z for z in cursor] 
    except cnx.ProgrammingError as err:
        if err.message == "no results to fetch":
            return None
        else:
            GPR.print_it2( label , "%r (%d)" %   (err.message ,   err.pgcode) , 0) # always print errors
    except cnx.IntegrityError as err:
        if err.pgcode == "23505":        # duplicate key
            GPR.print_it2( label , "%r (%d)" %   (err.message ,   err.pgcode) , 0) # always print errors
        else:
            GPR.print_it2( label , "%r (%d)" %   (err.message ,   err.pgcode) , 0) # always print errors     
    except cnx.InternalError as err:
        #InternalError: current transaction is aborted, commands ignored until end of transaction block
        GPR.print_it2( label , "%r (%d)" %   (err.message ,   err.pgcode) , 0) # always print errors
           
    except cnx.Error as err:
        GPR.print_it2( label , "%r (%d)" %   (err.message ,   err.pgcode) , 0 ) # always print errors
    finally:
        cursor.close()    

    return rel