def main():
db_connector = DBConnector('default')
curr=db_connector.getCursor()
conn=db_connector.getConn()
curr.execute("SELECT id,path FROM files_list where inserted=false and locked=false and pg_try_advisory_lock(tableoid::INTEGER,id) and path like '%contract%' limit 1")
[(id,path)] = curr.fetchall()
results=[]
while id > 0:
try:
dt = datetime.datetime.now()
curr.execute("update files_list set locked=true ,lock_time=now() where id=%s",(id,))
conn.commit()
print "--Try get file" , id , path
print "--RAISE warning 'Start to read file:%s:%s';" % (id,path)
dt = datetime.datetime.now()
parse_file_id(id,path)
except Exception as e:
print "--Poblem!",e
conn.rollback()
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)
print(exc_type, fname, exc_tb.tb_lineno)
traceback.print_exc()
finally:
curr.execute("update files_list set locked=false,inserted=true,insert_time=now() where id=%s; select pg_advisory_unlock(tableoid::INTEGER,id) from files_list where id = %s",(id,id))
conn.commit()
curr.execute("SELECT id,path FROM files_list where inserted=false and locked=false and pg_try_advisory_lock(tableoid::INTEGER,id) and path like '%contract%' limit 1")
[(id,path)] = curr.fetchall()
print "Seems, there is no more to insert."
print results
def writeToDb(self, choice=True):
self.db_connector = DBConnector('default')
self.dbConn = self.db_connector.getConn()
self.curr = self.db_connector.getCursor()
self.write_to_db = choice
import ftplib
import psycopg2
import sys
from helper import DBConnector
try:
db_conn = DBConnector('default')
cur = db_conn.getCursor()
cur.execute(
"CREATE TABLE files_list (id serial PRIMARY KEY, path varchar unique, insert_time timestamp);commit;"
)
del db_conn
except Exception, e:
print "Unable create table:", e
db_conn = DBConnector('default')
curr = db_conn.getCursor()
ftp = ftplib.FTP("ftp.zakupki.gov.ru")
ftp.login("free", "free")
files = []
def ftp_walk(ftp):
print 'Path:', ftp.pwd()
dirs = ftp.nlst()
for item in (path for path in dirs if path not in ('.', '..')):
try:
ftp.cwd(item)
print 'Changed to', ftp.pwd()
