def __init__(self,
input_file,
ip_position,
output_file=None,
ignore_lines=0):
'''
Runs processing from beginning to end.
@param input_file: path to CSV file containing the IP address column
@type input_file: str
@param ip_position: zero-origin index of column that contains the IP address.
@type ip_position: int
@param output_file: optional output file for the result CSV
@type output_file: {str | None|
@param ignore_lines: number of lines to copy to destination unprocessed. Used for headers.
@type ignore_lines: {int}
'''
if not os.access(input_file, os.R_OK):
print("Input file %s not found or not readable." % input_file)
sys.exit(1)
self.country_dict = IpCountryDict()
# Get lines to pass through, and array of row-arrays:
(headers, rows) = self.read_csvlines(input_file, ignore_lines)
self.write_country_state_city(headers, rows, ip_position, output_file)
def __init__(self, user, pwd):
self.ipCountryXlater = IpCountryDict()
self.user = user
self.pwd = pwd
self.db = MySQLDB(user=self.user, passwd=self.pwd, db='Edx')
self.db.dropTable(UserCountryTableCreator.DEST_TABLE)
self.db.createTable(
UserCountryTableCreator.DEST_TABLE,
OrderedDict({
'anon_screen_name': 'varchar(40) NOT NULL DEFAULT ""',
'two_letter_country': 'varchar(2) NOT NULL DEFAULT ""',
'three_letter_country': 'varchar(3) NOT NULL DEFAULT ""',
'country': 'varchar(255) NOT NULL DEFAULT ""'
}))
def __init__(self, user, pwd):
self.ipCountryXlater = IpCountryDict()
self.user = user
self.pwd = pwd
self.db = MySQLDB(user=self.user, passwd=self.pwd, db='Edx')
# Make sure table exists. It should, and it should be filled
# with all anon_screen_name and countries up the previous
# load:
self.db.createTable(
UserCountryTableCreator.DEST_TABLE,
OrderedDict({
'anon_screen_name': 'varchar(40) NOT NULL DEFAULT ""',
'two_letter_country': 'varchar(2) NOT NULL DEFAULT ""',
'three_letter_country': 'varchar(3) NOT NULL DEFAULT ""',
'country': 'varchar(255) NOT NULL DEFAULT ""'
}))
def __init__(self, user, pwd):
self.ipCountryXlater = IpCountryDict()
self.user = user
self.pwd = pwd
self.db = MySQLDB(user=self.user, passwd=self.pwd, db='Edx')
# Make sure table exists. It should, and it should be filled
# with all anon_screen_name and countries up the previous
# load:
createCmd = '''CREATE TABLE UserCountry (
anon_screen_name varchar(40) NOT NULL DEFAULT "",
two_letter_country varchar(2) NOT NULL DEFAULT "",
three_letter_country varchar(3) NOT NULL DEFAULT "",
country varchar(255) NOT NULL DEFAULT ""
) ENGINE=MyISAM;
'''
self.db.dropTable('UserCountry')
print("Creating table UserCountry...")
self.db.execute(createCmd)
print("Done creating table UserCountry.")
def __init__(self):
'''
Initializes extractor object with credentials from .ssh directory.
Set log file directory.
'''
home = expanduser("~")
userFile = home + '/.ssh/qualtrics_user'
tokenFile = home + '/.ssh/qualtrics_token'
dbFile = home + "/.ssh/mysql_user"
if os.path.isfile(userFile) == False:
sys.exit("User file not found: " + userFile)
if os.path.isfile(tokenFile) == False:
sys.exit("Token file not found: " + tokenFile)
if os.path.isfile(dbFile) == False:
sys.exit("MySQL user credentials not found: " + dbFile)
self.apiuser = None
self.apitoken = None
dbuser = None #@UnusedVariable
dbpass = None #@UnusedVariable
with open(userFile, 'r') as f:
self.apiuser = f.readline().rstrip()
with open(tokenFile, 'r') as f:
self.apitoken = f.readline().rstrip()
with open(dbFile, 'r') as f:
dbuser = f.readline().rstrip()
dbpass = f.readline().rstrip()
logging.basicConfig(
filename="EdxQualtricsETL_%d%d%d_%d%d.log" %
(dt.datetime.today().year, dt.datetime.today().month,
dt.datetime.today().day, dt.datetime.now().hour,
dt.datetime.now().minute),
level=logging.INFO)
self.lookup = IpCountryDict()
#************
MySQLDB.__init__(self, db="EdxQualtrics", user=dbuser, passwd=dbpass)
def __init__(self, ip_list_file):
countryDict = IpCountryDict()
with open(ip_list_file, 'r') as fd:
for ip in fd:
ip = ip.strip()
if len(ip) == 0:
print('%s\t%s' % ('<empty>', 'n/a'))
continue
if ip == '127.0.0.1':
print('%s\t%s\t' % (ip, 'localhost'))
continue
# If there are multiple, comma-separated ips, do them all:
ips = ip.split(',')
try:
for the_ip in ips:
country = countryDict.get(the_ip.strip(), 'n/a')
sys.stdout.write('%s\t%s' % (the_ip, country))
except ValueError as e:
print('%s\t%s' % (ip, 'n/a'))
continue
sys.stdout.write('\n')
def __init__(self, user, pwd):
self.ipCountryXlater = IpCountryDict()
self.user = user
self.pwd = pwd
self.db = MySQLDB(user=self.user, passwd=self.pwd, db='Edx')
# Make sure table exists. It should, and it should be filled
# with all anon_screen_name and countries up the previous
# load:
createCmd = '''CREATE TABLE %s (
anon_screen_name varchar(40) NOT NULL DEFAULT "",
two_letter_country varchar(2) NOT NULL DEFAULT "",
three_letter_country varchar(3) NOT NULL DEFAULT "",
country varchar(255) NOT NULL DEFAULT "",
region varchar(255) NOT NULL DEFAULT "",
city varchar(255) NOT NULL DEFAULT "",
lat_long point NOT NULL
) ENGINE=MyISAM;
''' % UserDetailedLocationTableCreator.DEST_TABLE
self.db.dropTable('UserCountry')
print("Creating table %..." %
UserDetailedLocationTableCreator.DEST_TABLE)
self.db.execute(createCmd)
print("Done creating table %s." %
UserDetailedLocationTableCreator.DEST_TABLE)
def setUp(self):
super(IpToCountryTester, self).setUp()
IpToCountryTester.lookup = IpCountryDict()
