def X_db_file_exists(cnx, in_dict): # , vol_id): gd = GetD(in_dict) gd['vol_id'] = in_dict['vol_id'].encode('utf8') select_query = ( "select 1 from files " "where vol_id = %(vol_id)r and folder_id = %(folder_id)s " "and file_name = %(file_name)r and file_mod_date = %(file_mod_date)r " ) # mysql.connector.errors.ProgrammingError: 1064 (42000): # You have an error in your SQL syntax; check the manual that corresponds # to your MySQL server version for the right syntax to use # near ''vol0003' and folder_id = 2 and # file_name = 'TV Show\xe2\x80\x94single' and file' at line 1 cursor = cnx.cursor() GPR.print_it(select_query % gd, 4) cursor.execute( select_query % gd ) r = [z for z in cursor] cursor.close() if r == [(1,)]: return True elif r == []: return False else: print "db_file_exists", r raise TypeError , ("db_file_exists says %r" % r)
def test_051_postgres(self): """Is the server running?""" # test mode: either the server is running or it isn't. both are ok as far as testing is concerned? # common error when no server is present: # note three addresses, all for "localhost" # could not connect to server: Connection refused # Is the server running on host "localhost" (::1) and accepting # TCP/IP connections on port 5432? # could not connect to server: Connection refused # Is the server running on host "localhost" (127.0.0.1) and accepting # TCP/IP connections on port 5432? # could not connect to server: Connection refused # Is the server running on host "localhost" (fe80::1) and accepting # TCP/IP connections on port 5432? try: cnx = psycopg2.connect(database='files', user='******', host='localhost' ) GPR.print_attrs("connect", cnx, verbose_level_threshold=1) # 4 except psycopg2.OperationalError as err: errmsg = 'could not connect to server: Connection refused\n\tIs the server running on host "localhost" (::1) and accepting\n\tTCP/IP connections on port 5432?\ncould not connect to server: Connection refused\n\tIs the server running on host "localhost" (127.0.0.1) and accepting\n\tTCP/IP connections on port 5432?\ncould not connect to server: Connection refused\n\tIs the server running on host "localhost" (fe80::1) and accepting\n\tTCP/IP connections on port 5432?\n' self.assertEqual ( err.message , errmsg )
def test_010_do_parse_args(self): s = u'/Users/donb/Ashley+Roberts/' argv = [] # argv = ["--help"]+[s] # argv = ["-rd 4"] argv = ["-d 2"] argv += ["-v"] # how tohave "-v" mean one and -v2, mean two? # argv += ["-v2"] # argv += ["-v"] # argv += ["-a"] argv += ["-p"] # argv += ["-f"] argv += [s] (options, args) = do_parse_args(argv) GPR.print_dict("options (after optparsing)", options.__dict__, left_col_width=24, verbose_level_threshold=2) self.assertEqual(options, Namespace(do_recursion=False, depth_limit=2, force_folder_scan=False, scan_hidden_files=False, scan_packages=True, verbose_level=2)) self.assertEqual( args, [u'/Users/donb/Ashley+Roberts/'] )
def do_push(in_value): """(push) (2 => 3) [(1, 40014149), (2, 42755279), (3, 45167012)]""" (depth, folder_file_id) = in_value # test for push of something other than just one. shouldn't happen. if prev_depth != len(in_stak): GPR.print_it1( "(push) prev_depth (%d) != len(in_stak) (%d)\n" % (prev_depth, len(in_stak)) ) GPR.print_it0( "(push) (%r" % (prev_depth,) ) in_stak.append(in_value) GPR.print_it0( "=> %r)" % ( depth) ) GPR.print_it1( "%r" % in_stak ) GPR.print_it1( "" ) tally["(push)"] +=1
def GetDR(item_dict, required_fields, quote_and_escape=True, verbose_level_threshold=3): """Convert from item_dict (Cocoa) forms to database-ready forms""" GPR.print_dict("GetDR(in)" , item_dict, 36, verbose_level_threshold) result_dict = {} for db_field_name in required_fields: if db_field_name in item_dict: dict_key_name = db_field_name else: try: db_field_name_index = map( lambda y: y[0], databaseAndURLKeys).index(db_field_name) dict_key_name = databaseAndURLKeys[db_field_name_index][1] except ValueError: dict_key_name = db_field_name # print "%16s => %-36s :" % (db_field_name, dict_key_name), # do special processing based on database field name, not on inherent type of argument? if db_field_name in ['vol_id', 'file_name', 'file_uti']: c = item_dict[dict_key_name].encode('utf8') if quote_and_escape: result_dict[db_field_name] = quote(escape(c)) else: result_dict[db_field_name] = c elif db_field_name in ['file_create_date', 'file_mod_date']: d = item_dict[dict_key_name] if isinstance( d, str): c = d else: c = str(db_df.stringFromDate_(d)) if quote_and_escape: result_dict[db_field_name] = quote(escape(c)) else: result_dict[db_field_name] = c elif db_field_name in ['file_size', 'file_id', 'folder_id']: # result_dict[db_field_name] = int(item_dict[dict_key_name]) else: result_dict[db_field_name] = item_dict[dict_key_name] GPR.print_dict_no_repr("GetDR(out)" + ( "(q+e)" if quote_and_escape else "(no q+e)"), result_dict, 36, verbose_level_threshold) return result_dict
def db_select_vol_idz(cnx, d): gd = GetD(d) select_query = ( "select vol_id from files " "where folder_id = 1 " "and file_name = %(file_name)s and file_create_date = %(file_create_date)s " ) cursor = cnx.cursor() GPR.print_it(select_query % gd, 4) cursor.execute( select_query , gd ) r = [z for z in cursor] if r == []: vol_id = None # print "vol_id is:", vol_id else: vol_id = r[0][0] # could have multiple results cursor.close() return vol_id
def vol_id_gen(cnx, in_gen): """processor for to find, store and then add vol_id to each item as it goes by""" # try: # select vol_id from volumes_uuid (based on volume uuid) # select vol_id from files (based on volume name and create date) # do a "test" insert of the volume entry just to get the autocreated vol_id back. (then act like it didn't happen :) local_vol_id = None for in_dict in in_gen: if local_vol_id == None: local_vol_id = db_get_vol_id(cnx, in_dict, local_vol_id) GPR.print_it2("vol_id_gen", "volume = %s, vol_id = %r" % (in_dict[NSURLNameKey], local_vol_id), verbose_level_threshold=2) in_dict['vol_id'] = local_vol_id yield in_dict GPR.print_it("end vol_id_gen.", verbose_level_threshold=3)
def execute_update_query(cnx, update_sql, sql_dict, label='execute_update_query', verbose_level_threshold=3): cursor = cnx.cursor() s = sqlparse.format(update_sql % sql_dict, reindent=True, encoding='utf8') GPR.print_it2( label, s, verbose_level_threshold) try: cursor.execute( update_sql % sql_dict) cnx.commit() except mysql.connector.Error as err: if err.errno == 1062 and err.sqlstate == '23000': if True or GPR.verbose_level >= n: n1 = err.msg.index('Duplicate entry') n2 = err.msg.index('for key ') msg2 = err.msg[n1:n2-1] print " "+repr(msg2) else: GPR.print_it2( label , "%r" % map(str , (err, err.errno , err.message , err.msg, err.sqlstate)), 0) # always print errors # Warning: It seems that you are trying to print a plain string containing unicode characters finally: cursor.close()
def db_connect(): """open and return mysql connector object""" default_dict = { 'user': '******', 'password': '', 'host': '127.0.0.1', 'database': 'files', 'buffered': True } config = ConfigParser.ConfigParser() config.readfp(open(CONFIG_FILE)) config_dict = dict(config.items('dbstuff')) default_dict.update(config_dict) GPR.print_dict("db_connect(default_dict+config_dict)", default_dict, verbose_level_threshold=1) # config.add_section('dbstuff') # for k,v in config_dict.items(): # self.__dict__.items(): # config.set('dbstuff', k, v) # with open(CONFIG_FILE, 'wb') as configfile: # config.write(configfile) # GPR.print_dict("db_connect(default_dict)", default_dict, verbose_level_threshold=1) try: cnx = mysql.connector.connect(**default_dict) except mysql.connector.Error as err: if err.errno == errorcode.ER_ACCESS_DENIED_ERROR: print("Username or password %r and %r?" % (default_dict['user'], default_dict['password'])) elif err.errno == errorcode.ER_BAD_DB_ERROR: print "Database %r does not exist." % default_dict['database'] else: print 'err:', err GPR.print_attrs("mysql.connector", cnx, verbose_level_threshold=4) return cnx
def db_connect(verbose_level_threshold=2): """open and return mysql connector object""" default_dict = { 'user': '******', 'password': '', 'host': '127.0.0.1', 'database': 'files' } try: config = ConfigParser.ConfigParser() config.readfp(open(CONFIG_FILE)) config_dict = dict(config.items('postgres')) default_dict.update(config_dict) except ConfigParser.NoSectionError as err: print err GPR.print_dict("postgres", default_dict, verbose_level_threshold=3) try: cnx = psycopg2.connect(**default_dict) # connection could have a cursor factory? GPR.print_attrs("connect", cnx, verbose_level_threshold=3) except psycopg2.OperationalError as err: if err.message.startswith( 'could not connect to server: Connection refused') : print "Is the postgres server running?" else: print err.message return return cnx # config.add_section('postgres') # for k,v in config_dict.items(): # self.__dict__.items(): # config.set('postgres', k, v) # with open(CONFIG_FILE, 'wb') as configfile: # config.write(configfile) # GPR.print_dict("db_connect(default_dict)", default_dict, verbose_level_threshold=1) try: cnx = mysql.connector.connect(**default_dict) except mysql.connector.Error as err: if err.errno == errorcode.ER_ACCESS_DENIED_ERROR: print("Username or password %r and %r?" % (default_dict['user'], default_dict['password'])) elif err.errno == errorcode.ER_BAD_DB_ERROR: print "Database %r does not exist." % default_dict['database'] else: print 'err:', err GPR.print_attrs("mysql.connector", cnx, verbose_level_threshold=4) return cnx
def X_GetD(item_dict): """Convert from item_dict (Cocoa) forms to something that the database DBI can convert from""" d = {} for dk, fk in databaseAndURLKeys: if dk: if fk in [NSURLNameKey, NSURLTypeIdentifierKey]: d[dk] = item_dict[fk].encode('utf8') elif dk in ['file_create_date', 'file_mod_date']: d[dk] = str(item_dict[fk])[:-len(" +0000")] # "2011-07-02 21:02:54 +0000" => "2011-07-02 21:02:54" elif type(item_dict[fk]) == objc._pythonify.OC_PythonLong: # print """"type(item_dict[fk]) = objc._pythonify.OC_PythonLong""", item_dict[fk], int(item_dict[fk]) d[dk] = int(item_dict[fk]) # elif 'objc._pythonify.OC_PythonLong' in str(type(item_dict[fk])): # print """"nscfnumber" in str(type(item_dict[fk]):""", item_dict[fk], int(item_dict[fk]), objc._pythonify.OC_PythonLong # d[dk] = int(item_dict[fk]) else: # print type(item_dict[fk]) d[dk] = item_dict[fk] GPR.print_dict("GetD result", d, 32, 4) return d
def db_insert_volume_data(cnx, vol_id, volume_url): """ insert/update volumes table with volume specific data, eg uuid, total capacity, available capacity """ # get volume info values, error = volume_url.resourceValuesForKeys_error_( ['NSURLVolumeUUIDStringKey', 'NSURLVolumeTotalCapacityKey', 'NSURLVolumeAvailableCapacityKey', 'NSURLVolumeSupportsPersistentIDsKey', 'NSURLVolumeSupportsVolumeSizesKey'] , None ) if error is not None: print print error # volume_dict.update(dict(values)) dv = dict(values) GPR.print_dict("volume info", values, 36, 1) # note: "on duplicate key update" of vol_total_capacity and vol_available_capacity. query = ("insert into volume_uuids " "(vol_id, vol_uuid, vol_total_capacity, vol_available_capacity) " "values ( %s, %s, %s, %s ) " "on duplicate key update " "vol_total_capacity = values(vol_total_capacity), " "vol_available_capacity = values(vol_available_capacity)" ) data = (vol_id, str(dv['NSURLVolumeUUIDStringKey']) , int(dv['NSURLVolumeTotalCapacityKey']), int(dv['NSURLVolumeAvailableCapacityKey']) ) (l , vol_id) = execute_insert_query(cnx, query, data, 1) GPR.pr4(l, vol_id, "", data[1], 1)
def Z_db_query_folder(cnx, item_dict): """get database contents of item as folder.""" vol_id = item_dict['vol_id'].encode('utf8') this_folder_id = item_dict['NSFileSystemFileNumber'] # in sql format "%r" is good because it provides quotes but will also insert u'vol0001' # so either: str() the text (or utf8 encode it?) or format it like '%s' (but this is dumb quotes) sql = "select vol_id, folder_id, file_name, file_id, file_mod_date from files "+\ "where vol_id = %r and folder_id = %d " data = (vol_id, this_folder_id ) cur = cnx.cursor(cursor_class=MySQLCursorDict) GPR.print_it( sql % data, verbose_level_threshold=2) cur.execute( sql % data ) cur.set_rel_name(in_rel_name="files_del") # need name at relation init time (ie, inside cursor fetch) r = cur.fetchall() cur.close() return r
def db_connect_psycopg2(): cnx = psycopg2.connect(database='files', user='******', host='localhost' ) GPR.print_attrs("psycopg2.connect", cnx, verbose_level_threshold=1) # 4 return cnx
def do_pop(): """ (pop) (3 => 2) [(1, 40014149), (2, 42755279)]""" # test for pop of something other than just one. can happen. is okay:) if depth+1 != len(in_stak): GPR.print_it1( " (pop) depth+1 (%d) != len(in_stak) (%d)\n" % (depth+1, len(in_stak)) ) GPR.print_it0( " (pop) (%r"% (len(in_stak), ) ) tally["(pop)"] +=1 (d,ffid) = in_stak.pop() GPR.print_it0( "=> %r) "% (len(in_stak), ) ) # this should really be a method on a subclass of stak? if hasattr(in_stak, 'RS'): if (d,ffid) in in_stak.RS: # in_stak.RS.keys(): GPR.print_it0( "key %r in stak: %r" % ((d,ffid), in_stak) ) GPR.print_it1( "(dict)") GPR.print_it1("") return in_stak.RS.pop((d,ffid)) # in_stak.RS[(d,ffid)] else: # not error, directory was up to date(?) GPR.print_it1( "key %r not in stak: %r" % ((d,ffid), in_stak) ) GPR.print_it1("") return [] else: GPR.print_it0( "in_stak has no attr 'RS'" ) GPR.print_it1( "%r" % in_stak ) GPR.print_it1("") return []
def main(): # some favorite testing files s = '/Volumes/Ulysses/bittorrent' s = u'/Users/donb/Downloads/incomplete' s = '/Volumes/Ulysses/TV Shows/Nikita/' # package s = u"/Users/donb/Documents/Installing Evernote v. 4.6.2—Windows Seven.rtfd" s = '/Volumes/Ulysses/bittorrent' s = u'/Users/donb/Downloads/incomplete' s = u'/Users/donb/Ashley+Roberts/' s = "/Volumes/Taos/videogame/" s = "/Volumes/Daytona/TV Series/Americas Next Top Model" s = '.' s = "/Volumes/Taos/videogame/Perfect Dark/Joanna Dark/" s = u'/Users/donb/Ashley+Roberts/' s = u"~/Catherine Video Review.mp4" s = "/Volumes/Brandywine/TV Show—single/" s = '/Volumes/Ulysses/TV Shows/Nikita/' s = '/Volumes/Ulysses/bittorrent/' s = "/Volumes/Corinna" s = "/Volumes/Corinna/Actress/Alison Armitage" s = "/Volumes/Corinna/Actress/Keira Knightley/Keira Knightley - Kenneth Willardt's GQ Photoshoot" s = "/Volumes/Dunharrow" # hack to have Textmate run with hardwired arguments while command line can be free… if os.getenv('TM_LINE_NUMBER' ): argv = [] argv += ["-v"] # verbose_level = 2 argv += ["-v"] # argv += ["-v"] # verbose_level = 4 # argv = ["-d 3"] argv += ["-f"] # force folder scan # argv += ["-p"] # scanning packages argv += [s] else: argv = sys.argv[1:] (options, args) = do_parse_args(argv) # no args means do the current directory if args == []: args = ["."] args2 = [] for a in args: try: unicode(a) except UnicodeDecodeError: a2 = a.decode('utf8') # print "arg [ %s ] is a unicode string" % (a2, ) GPR.print_it2("arg is a unicode string", a2, verbose_level_threshold=1) args2.append(a2) else: args2.append(a) args = args2 args = [os.path.abspath(os.path.expanduser(a)) for a in args] GPR.verbose_level = options.verbose_level GPR.print_list("sys.argv", sys.argv, verbose_level_threshold=3) # display list of timezones if options.verbose_level >= 4: print_timezones("time_zones") GPR.print_dict("options (after optparsing)", options.__dict__, left_col_width=24, verbose_level_threshold=2) GPR.print_list("args (after optparsing)", args, verbose_level_threshold=3) do_args(args, options)
def do_args(args, options): """do_args is the high-level, self-contained routine most like the command-line invocation""" cnx = db_connect() required_fields = ['vol_id', 'folder_id', 'file_name', 'file_id', 'file_size', 'file_create_date', 'file_mod_date', 'file_uti' ] try: for basepath in args: for arg_dict in do_arg_gen(basepath, cnx, options): # depth 0 should be a fully-realized level if (arg_dict['depth'] < 0): if not ('directory_is_up_to_date' in arg_dict) or not arg_dict['directory_is_up_to_date']: sql_dict = GetDR(arg_dict, required_fields) sql_dict['file_mod_date'] = "'1970-01-01 00:00:00'" # args are escaped and quoted at this point add_file_sql = ("insert into files " "(vol_id, folder_id, file_name, file_id, file_size, file_create_date, file_mod_date, file_uti) " "values " "( %(vol_id)s, %(folder_id)s, %(file_name)s, %(file_id)s, %(file_size)s, %(file_create_date)s, " "%(file_mod_date)s, %(file_uti)s ) " ) # execute_update_query(cnx, add_file_sql , sql_dict, label='(depth < 0)', verbose_level_threshold=3 ) db_execute_sql(cnx, add_file_sql % sql_dict, label='(depth < 0)', verbose_level_threshold=2) GPR.pr7z( arg_dict ) elif 'sql_action' in arg_dict: if arg_dict['sql_action'] in ["update_directory", "insert"]: # technically, we are updating (ie, completing) the directory # before we do the directory entries? consistency problem if we fail? add_file_sql = ("insert into files " "(vol_id, folder_id, file_name, file_id, file_size, file_create_date, file_mod_date, file_uti) " "values " "( %(vol_id)s, %(folder_id)s, %(file_name)s, %(file_id)s, %(file_size)s, %(file_create_date)s, " "%(file_mod_date)s, %(file_uti)s ) " ) # execute_update_query(cnx, add_file_sql , sql_dict, label=arg_dict['sql_action'], verbose_level_threshold=2) # sql and dict are "%"'ed inside function db_execute(cnx, add_file_sql, arg_dict, required_fields, label="do_args" + arg_dict['sql_action'], verbose_level_threshold=2) else: sql_dict = GetDR(arg_dict, required_fields) GPR.print_it(add_file_sql % sql_dict, 3) GPR.pr7z( arg_dict ) elif (arg_dict['depth'] == 0): GPR.pr7z( arg_dict , verbose_level_threshold=1) else: GPR.pr7z( arg_dict , verbose_level_threshold=2) except MyError, err: print err.description
def do_arg_gen(basepath, cnx, options): x_gen = files_generator(basepath, options) my_stak = stak() # contains self.RS fs_gen = partial(files_stak_gen, in_stak=my_stak, cnx=cnx) y_gen = partial(vol_id_gen, cnx) # for fs_dict in y_gen( fs_gen( x_gen ) ): for fs_dict in fs_gen( y_gen( x_gen ) ): depth = fs_dict['depth'] file_id = fs_dict['NSFileSystemFileNumber'] if is_a_directory(fs_dict, options): GPR.print_it0( "(directory)" ) if (depth < 0 ): GPR.print_it1( "(depth < 0)" ) # eol else: dir_is_up_to_date = not options.force_folder_scan and db_file_exists(cnx, fs_dict) fs_dict['directory_is_up_to_date'] = dir_is_up_to_date if (dir_is_up_to_date): GPR.print_it1( "(up_to_date)" ) # eol else: GPR.print_it0( "(to_be_scanned)" ) GPR.print_it0( "(scanning)" ) rel = db_query_folder(cnx, fs_dict) GPR.print_it0( "(len=%r)" % (len(rel),) ) GPR.print_it0( "(storing at) %r" % ( (depth+1, file_id) ,) ) # don't store those of zero length because you'll never pop them? no: # do store zero lengths because we will want to "subtract against them" in directory check. my_stak.RS[ (depth+1, file_id) ] = rel GPR.print_it1( "stak: %r" % (my_stak,) ) # eol fs_dict['sql_action'] = "update_directory" GPR.print_it0( "(update directory)" ) folder_id = fs_dict['NSFileSystemFolderNumber'] fs_dict['current_item_directory_is_being_checked'] = (depth, folder_id) in my_stak.RS if fs_dict['current_item_directory_is_being_checked']: GPR.print_it0( "(check against container directory %r)" % ((depth,folder_id ) ,) , verbose_level_threshold=3) vol_id = fs_dict['vol_id'] filename = fs_dict[NSURLNameKey] # still unicode, encode late. required_fields = ['vol_id', 'folder_id', 'file_name', 'file_id', 'file_mod_date' ] sql_dict = GetDR(fs_dict, required_fields, quote_and_escape=False, verbose_level_threshold=4) #ie, not for sql, just for values rs = ( vol_id, folder_id, filename, file_id, sql_dict['file_mod_date']) rs2 = tuple([filename if k == 'file_name' else sql_dict[k] for k in required_fields]) # unicode filename if rs != rs2: print "\n%r != %r\n" % (rs, rs2) if rs in my_stak.RS[ (depth,folder_id ) ]: my_stak.RS[ (depth,folder_id ) ] -= rs GPR.print_it0( "(ignore)" ) GPR.print_it0( "(already in database) %r" % my_stak ) else: fs_dict['to_be_inserted'] = True fs_dict['sql_action'] = "insert" GPR.print_it0( "(insert)" ) # # try: # my_stak.RS[ (depth,folder_id ) ] -= rs # my_stak.RS[ (depth,folder_id ) ]._convert_to_row(rs) , or just, tuple(in_row) # GPR.print_it0( "(ignore)" ) # GPR.print_it0( "(already in database) %r" % my_stak ) # except KeyError: # ie, the key of rs is not in the set of stak.ts[(d,fid)] # fs_dict['to_be_inserted'] = True # fs_dict['sql_action'] = "insert" # GPR.print_it0( "(insert)" ) yield fs_dict # end gen if options.verbose_level >= 3: print "end do_arg_gen. my_stak is", my_stak
def files_generator(basepath, options): """a generator which yields all files (as file_dicts) including volume, superfolder(s), basepath, and then all subfiles (subject to depth_limit and enumerator options). """ GPR.print_it25("files_generator", basepath, 2) superfolders_list = [] basepath_url = NSURL.fileURLWithPath_(basepath) # begin loop going upwards url = NSURL.fileURLWithPath_(basepath) while True: d1 = GetURLValues(url, enumeratorURLKeys) superfolders_list.insert(0,d1) if d1[NSURLIsVolumeKey]: break # go "upwards" one level (towards volume) url = url.URLByDeletingLastPathComponent() GPR.print_superfolders_list("volume, superfolder(s)", superfolders_list, 4) # now go back down, yielding dict objects at each step: n = len(superfolders_list) for i, superfolder_dict in enumerate(superfolders_list): superfolder_dict['depth'] = i+1-n yield superfolder_dict # last dict in superfolder list is the basepath_dict basepath_dict = superfolder_dict item_is_package = is_item_a_package(basepath_url) if basepath_dict[NSURLIsDirectoryKey] and item_is_package and not options.scan_packages: GPR.print_it("\nbasepath is a directory and a package but we're not scanning packages.\n", 1) return # we've yielded basepath above, don't enumerate if basepath is not a directory (or package and we want packages) if basepath_dict[NSURLIsDirectoryKey]: enumeratorOptionKeys = 0L if not options.scan_packages: enumeratorOptionKeys |= NSDirectoryEnumerationSkipsPackageDescendants if not options.scan_hidden_files: enumeratorOptionKeys |= NSDirectoryEnumerationSkipsHiddenFiles enumerator = sharedFM.enumeratorAtURL_includingPropertiesForKeys_options_errorHandler_( basepath_url, enumeratorURLKeys, enumeratorOptionKeys, error_handler_for_enumerator ) for url in enumerator: item_dict = GetURLValues(url, enumeratorURLKeys) depth = enumerator.level() item_dict['depth'] = depth if options.depth_limit and (depth >= options.depth_limit-1): enumerator.skipDescendants() yield item_dict GPR.print_it2("end files_generator", basepath, verbose_level_threshold=3)
def db_execute_relation(cnx, sql_query, item_dict, required_fields, label, verbose_level_threshold=2): """db_execute is GetDR, sqlparse, cursor.execute, return rel.add( r ) for r in cursor and cursor.close """ sql_dict = GetDR(item_dict, required_fields, verbose_level_threshold=verbose_level_threshold) sql_query = sql_query % sql_dict s = sqlparse.format( sql_query, reindent=True, encoding='utf8') GPR.print_it2( label, s, verbose_level_threshold) # 4 try: cursor = cnx.cursor() cursor.execute( sql_query ) column_names = [column.name for column in cursor.description] rel = relation( column_names, [] , "pg_del_files") for r in cursor: l = list(r) l[2] = l[2].decode('utf8') # file_name l[4] = str(l[4]) rel.add( l ) cnx.commit() # r = [z for z in cursor] except cnx.ProgrammingError as err: if err.message == "no results to fetch": return None else: GPR.print_it2( label , "%r (%d)" % (err.message , err.pgcode) , 0) # always print errors except cnx.IntegrityError as err: if err.pgcode == "23505": # duplicate key GPR.print_it2( label , "%r (%d)" % (err.message , err.pgcode) , 0) # always print errors else: GPR.print_it2( label , "%r (%d)" % (err.message , err.pgcode) , 0) # always print errors except cnx.InternalError as err: #InternalError: current transaction is aborted, commands ignored until end of transaction block GPR.print_it2( label , "%r (%d)" % (err.message , err.pgcode) , 0) # always print errors except cnx.Error as err: GPR.print_it2( label , "%r (%d)" % (err.message , err.pgcode) , 0 ) # always print errors finally: cursor.close() return rel