def get_conn_sqlite(): conn = sqliteDefaults.get_conn(dbname+".db") if(conn): print "\n\n\tSuccessfully Connected.\n\n" else: print "\n\n\tERROR: Unable to establish connection" return conn
def get_conn_sqlite():
conn = sqliteDefaults.get_conn(dbname + ".db")
if (conn):
print "\n\n\tSuccessfully Connected.\n\n"
else:
print "\n\n\tERROR: Unable to establish connection"
return conn
def init_db_sqlite():
conn = sqliteDefaults.get_conn(dbname + ".db")
conn.execute('''Create table if not exists website_regex(
base_url TEXT,
exec_code TEXT,
date_of_addition DATE,
primary key(base_url, date_of_addition)
);
''')
conn.commit()
conn.close()
def init_db_sqlite():
conn = sqliteDefaults.get_conn(dbname+".db")
conn.execute('''Create table if not exists website_regex(
base_url TEXT,
exec_code TEXT,
date_of_addition DATE,
primary key(base_url, date_of_addition)
);
''')
conn.commit()
conn.close()
def connectToSQLiteDB(self, dbFilePath="GoogleSearchResults.db", dbTableName="SearchResultURLs", printing=True):
"""
Args:
dbFilePath: the file path of the SQLite database file. If not a .db file, it is corrected.
e.g. "xxx/xxx/xxx.db" stays the same, whereas "xxx/xxx/xxx" becomes the former, and "xxx/xxx/" (i.e. a directory) bedomes "xxx/xxx/GoogleSearchResults.db"
dbTableName: the SQLite table name to be referred to henceforth.
printing: if we should print to terminal or not.
Returns: True or False, depending on whether we have successfully connected to SQLite and created a usable table, or not.
"""
## Correct common errors:
if dbFilePath.endswith("/") or dbFilePath.endswith("\\"):
dbFilePath+="GoogleSearchResults.db"
if not dbFilePath.endswith(".db"):
dbFilePath+=".db"
try:
self.conn=sqliteDefaults.get_conn(dbFilePath, printing)
except Exception, e:
print_error(printing, self.__class__.__name__, sys._getframe().f_code.co_name, "could not connect to SQLite database.", e)
self.conn = None
return False
##-------CODE FOR GETTING A RESULTS FOR DATE-STAGGERED QUERIES AS AN SQLITE DATABASE-------#
signal.signal(signal.SIGINT, ctrl_c_signal_handler) ## assign ctrl_c_signal_handler to Ctrl+C, i.e. SIGINT
conn=sqliteDefaults.get_conn(db_file_path)
conn.execute('''Create table if not exists %s(
Topic TEXT,
StartDate INTEGER,
EndDate INTEGER,
ResultPageNumber INTEGER,
URL TEXT,
ResultNumber INTEGER,
PRIMARY KEY(Topic, URL)
);
'''%db_table_name)
conn.commit()
initial_start_date = to_julian_date_datetime(datetime.now().date())
10 * (int(results_per_page * (540 / float(80)) / 10)))
elif results_per_page == 100 and wait_between_pages / float(
results_per_page) < 600 / float(100):
print "\n\t\tWARNING: the wait time between pages may not be large enough to prevent IP blocking."
print "\t\tRecommend wait time between pages: %s seconds or more." % (
10 * (int(results_per_page * (600 / float(100)) / 10)))
print "\n\n\n\n"
##-------CODE FOR GETTING A RESULTS FOR DATE-STAGGERED QUERIES AS AN SQLITE DATABASE-------#
signal.signal(signal.SIGINT, ctrl_c_signal_handler
) ## assign ctrl_c_signal_handler to Ctrl+C, i.e. SIGINT
conn = sqliteDefaults.get_conn(db_file_path)
conn.execute('''Create table if not exists %s(
Topic TEXT,
StartDate INTEGER,
EndDate INTEGER,
ResultPageNumber INTEGER,
URL TEXT,
ResultNumber INTEGER,
PRIMARY KEY(Topic, URL)
);
''' % db_table_name)
conn.commit()
initial_start_date = to_julian_date_datetime(datetime.now().date())
start_date = 0
end_date = 0
import sqliteDefaults
import os
import extraction_text_manip
os.system("reset")
conn = sqliteDefaults.get_conn("article_extract_db.db")
conn.execute('''CREATE TABLE IF NOT EXISTS `articles_clean` (
`company_or_sector` TEXT,
`article_url` TEXT,
PRIMARY KEY(company_or_sector, article_url)
);
''')
conn.commit()
company_name = 'Infosys'
articles = sqliteDefaults.verified_select_sqlite(conn,
"SELECT DISTINCT article_url, company_name, article_headline, article_text, article_date \
FROM articles \
WHERE company_name='%s' \
and article_url not in (select article_url from articles_clean)\
ORDER BY article_url ASC\
"%(company_name)
)
conn2 = sqliteDefaults.get_conn("extracted_search_urls.db")
company_dict = {}
temp_table = sqliteDefaults.verified_select_sqlite(conn2,"SELECT DISTINCT ArticleTopic from articleUrls order by ArticleTopic asc")
for i in range(0,len(temp_table)):
