class Csv:
def __init__(self, fileName=None):
self.logger = LogManager(__name__)
if fileName is not None:
self.writer = csv.writer(open(fileName, 'a'), dialect='excel')
def writeCsvRow(self, data):
try:
self.writer.writerow(data)
except Exception, x:
self.logger.error(x)
class Utils:
def __init__(self):
self.logger = LogManager(__name__)
def downloadFile(self, url, savePath):
try:
directory = os.path.dirname(savePath)
if not os.path.exists(directory):
os.makedirs(directory)
webFile = urllib2.urlopen(url)
localFile = open(savePath, 'wb')
localFile.write(webFile.read())
except Exception, x:
self.logger.error(x)
class Spider:
def __init__(self):
self.logger = LogManager(__name__)
self.opener = None
def login(self, url, loginInfo, retry=0):
"""
Login request for user
url = '' Ex. http://www.example.com/login
loginInfo = {} Ex. {'user': '******', 'pass': '******'}
"""
self.opener = self.createOpener([config.USER_AGENT], self.createCookieJarHandler())
urllib2.install_opener(self.opener)
try:
return self.opener.open(url, urllib.urlencode(loginInfo)).read()
except Exception, x:
self.logger.error(x.message)
if retry < config.RETRY_COUNT:
self.login(url, loginInfo, retry + 1)
return None
def __init__(self):
self.logger = LogManager(__name__)
def __init__(self, fileName=None):
self.logger = LogManager(__name__)
if fileName is not None:
self.writer = csv.writer(open(fileName, 'a'), dialect='excel')
def __init__(self):
self.logger = LogManager(__name__)
self.opener = None