class URLRoller():
def __init__(self):
self.file_logger = FileLogger()
cfg_parser = CfgFileParser()
self.Config = cfg_parser.file_reader(CONFIGFILE)
self.Project = self.Config['Collector']['project']
self.step = int(self.Config[self.Project]['loader'])
enum_list = []
for Idx in range(self.step):
print('in the loop')
col_data = self.Config[self.Project ][str(Idx+1)]
file_in = codecs.open(col_data, 'r', 'utf-8')
Cpt = 0
for Line in file_in:
URL = Line[:-1].strip()
URL = re.sub(r'\s+', ' ', URL)
enum_list.append(URL)
Cpt +=1
self.file_logger.csv_log('URL %s:' % Cpt, URL)
self.URLList = enum_list
self.StartURL = enum_list[0]
self.URLRoller = itertools.cycle(enum_list)
def next(self):
return next(self.URLRoller)
def starter(self):
return self.StartURL
class LocationCalculator:
def __init__(self):
self.Logger = FileLogger()
def Locate(self, RetrivedData):
try:
self.Logger.Info(RetrivedData)
return self.GetRSS(RetrivedData)
except Exception as e:
self.Logger.Error("ERROR in LocationCalculator: "+str(e))
def GetRSS(self, RetrivedData):
beaconsRSS = []
for beacon in RetrivedData:
if not any(b['UID'] == beacon['UID'] for b in beaconsRSS):
beaconsFound = [x for x in RetrivedData if x['UID'] == beacon['UID']]
totalRSS =0
for b in beaconsFound:
if "RSS" in b:
totalRSS += int(b['RSS'])
newBeaconRSS = {"UID": beacon['UID'], "RSS": totalRSS/len(beaconsFound)}
beaconsRSS.append(newBeaconRSS)
self.Logger.Info(beaconsRSS)
return beaconsRSS
class JSONPipeline(object):
# Defines a class to save the scraped data into a json line file
@classmethod
def from_crawler(cls, crawler):
pipeline = cls()
crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
return pipeline
def spider_opened(self,spider):
# Open the file and start to export data
self.file_logger = FileLogger()
self.file_logger.csv_log('PIPELINE', 'open a JSON pipeline')
self.file = codecs.open('output.json', 'wb','utf-8')
self.exporter = JsonLinesItemExporter(self.file)
self.exporter.start_exporting()
def spider_closed(self,spider):
# Stop exporting data and close the file
self.exporter.finish_exporting()
self.file.close()
def process_item(self,item,spider):
# Export the data scraped by the spider
self.exporter.export_item(item)
return item
def __init__(self, profile_url='', next=False, *args, **kwargs):
# Get value for given attributes
super(ThesesSpider, self).__init__(*args, **kwargs)
self.start_urls = [profile_url]
self.Next = next
# Instantiate a FileLogger object
self.file_logger = FileLogger()
def spider_opened(self,spider):
# Open the file and start to export data
self.file_logger = FileLogger()
self.file_logger.csv_log('PIPELINE', 'open a JSON pipeline')
self.file = codecs.open('output.json', 'wb','utf-8')
self.exporter = JsonLinesItemExporter(self.file)
self.exporter.start_exporting()
def __getChainOfLoggers():
errorLogger = ErrorLogger(AbstractLogger.ERROR)
fileLogger = FileLogger(AbstractLogger.DEBUG)
consoleLogger = ConsoleLogger(AbstractLogger.INFO)
errorLogger.setNextLogger(fileLogger)
fileLogger.setNextLogger(consoleLogger)
return errorLogger
class MongoPipeline(object):
# Defines a class to save the scraped data into MongoDB
def spider_opened(self,spider):
# Get the database parameters from the .cfg file
cfg_parser = CfgFileParser()
self.Config = cfg_parser.file_reader(CONFIGFILE)
# Assign the mongoDB information
self.file_logger = FileLogger()
self.Project = self.Config['Collector']['project']
self.mongo_db = self.Config[self.Project]['mongo_db']
self.Host = self.Config[self.Project]['host']
self.Port = int(self.Config[self.Project]['port'])
self.Collection = self.Config[self.Project]['collection']
self.DataSet = self.Config[self.Project]['dataset']
self.mongo_client = MongoClient(self.Host, self.Port)
self.DBase = self.mongo_client[self.mongo_db]
self.DBase[self.Collection].ensure_index([('idx', pymongo.ASCENDING),('created_at', pymongo.ASCENDING)])
self.Idx = 0
Cursor = self.DBase[self.Collection].find({}).sort([('created_at',pymongo.DESCENDING)]).limit(1)
for Record in Cursor :
self.Idx = Record['idx']
self.file_logger = FileLogger()
self.file_logger.csv_log('PIPELINE', 'open a MONGO pipeline')
@classmethod
def from_crawler(cls, crawler):
pipeline = cls()
crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
# crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
return pipeline
def spider_closed(self,spider):
self.mongo_client.close()
def process_item(self, item, spider):
# Export the data scraped by the spider
# Get the dom as a str from the scrapy.Item
dom_list = item['dom_tree']
item_comp = BinaryCompressor.compress(dom_list[0])
self.Idx +=1
self.file_logger.csv_log('INFO', 'process_item', self.Idx)
Record = {'created_at': datetime.datetime.now(),
'parsed': False,
'profile_url': item['profile_url'],
'page_url': item['page_url'],
'level': item['level'],
'zipdom': item_comp,
'dataset':self.DataSet,
'idx':self.Idx}
self.DBase[self.Collection].insert(Record)
return item
class Extractor(yapdi.Daemon):
def __init__(self):
self.file_logger = FileLogger()
cfg_parser = CfgFileParser()
self.Config = cfg_parser.file_reader(cfg_file)
pid_file = self.Config['Extractor']['pid']
self.Wait = int(self.Config['Extractor']['wait'])
self.file_logger = FileLogger()
self.Project = self.Config['Extractor']['project']
Parser = self.Config[self.Project]['parser']
self.Parser = eval(Parser + '()')
self.Parser.Project = self.Project
self.Parser.init_DataBase()
try:
Restriction = self.Config['Extractor']['restriction']
self.Restriction = json.loads(Restriction)
self.file_logger.csv_log('EXTRACTOR', 'Extractor init ok')
except Exception as exp:
self.file_logger.csv_log('ERROR',exp)
raise
yapdi.Daemon.__init__(self, pid_file)
def Extracting(self):
self.file_logger.csv_log('EXTRACTOR', 'start Extracting')
while True:
try:
self.Parser.parse(self.Restriction)
time.sleep(self.Wait)
except Exception as exp:
self.file_logger.csv_log('EXTRACTOR', 'Bug in Extracting():', exp)
class DataManager:
def __init__(self):
self.DBConnection = DBRetriever()
self.Logger = FileLogger()
self.Locator = LocationCalculator()
self.QueryBuilder = QueryBuilder()
def Manage(self, parameters):
# BuildQuery
query = self.QueryBuilder.Build(parameters)
# IfDeviceID
if 'DeviceID' in query:
return self.GetDeviceLocation(query.pop('DeviceID'), query)
# IfBeaconID
if 'BeaconID' in query:
self.Logger.Info('Datamanager: ' + str(query))
return self.GetBeaconinformation('BeaconInformation', query)
def GetDeviceLocation(self, deviceID, filter):
nearByBeacons = self.Querydatabase(deviceID, filter)
return self.Locator.Locate(nearByBeacons)
def GetBeaconinformation(self, beaconInformation, filter):
return self.Querydatabase(beaconInformation, filter)
def Querydatabase(self, collectionName, filter):
return self.DBConnection.QueryDatabase(collectionName, filter)
def return_result(self, selector: int):
switcher = {
1: ConsoleLogger(),
2: FileLogger(),
3: DatabaseLogger()
}
func = switcher.get(selector)
return func
def __init__(self):
cfg_parser = CfgFileParser()
self.Config = cfg_parser.file_reader(CONFIGFILE)
self.Project = self.Config['Collector']['project']
self.Drop = self.Config[self.Project]['drop'] == 'True'
self.Cycle = self.Config[self.Project]['cycle'] == 'True'
self.Next = self.Config[self.Project]['next'] == 'True'
self.mongo_db = self.Config[self.Project]['mongo_db']
self.Host = self.Config[self.Project]['host']
self.Port = int(self.Config[self.Project]['port'])
self.Collection = self.Config[self.Project]['collection']
self.mongo_client = MongoClient(self.Host, self.Port)
self.DBase = self.mongo_client[self.mongo_db]
if self.Drop:
self.DBase[self.Collection].drop()
self.file_logger = FileLogger()
self.set_up()
def get_log(self, msg, type):
if type is 'Console':
return ConsoleLogger(msg)
elif type is 'File':
return FileLogger(msg)
elif type is 'Database':
return DatabaseLogger(msg)
def requestData(self,
file_log_name=None,
referer=None,
x_requested_with=None,
content_type=None,
**urllibArg):
try:
# This is the Request
req = urllib.request.Request(**urllibArg)
if referer is not None:
req.add_header('Referer', referer)
if x_requested_with is not None:
req.add_header('X-Requested-With', x_requested_with)
if content_type is not None:
req.add_header('Content-Type', content_type)
print("Request -> " + urllibArg['url'])
site = self.opener.open(req)
response_data = None
if site.info().get('Content-Encoding') == 'gzip':
response_data = gzip.decompress(site.read())
else:
response_data = site.read()
if file_log_name is not None:
fileLogger = FileLogger(file_log_name=file_log_name,
data=response_data,
mode="wb")
except Exception as e:
response_data = None
reference = urllibArg['url']
message = str(urllibArg)
timestampString = datetime.fromtimestamp(time()).strftime('%b-%H')
fileLogger = FileLogger(file_log_name=('request-data-' +
timestampString + '.txt'),
reference=reference,
data=message,
exception=e,
mode="a")
return response_data
def __init__(self):
self.file_logger = FileLogger()
cfg_parser = CfgFileParser()
self.Config = cfg_parser.file_reader(CONFIGFILE)
self.Agenda = []
self.Pace = self.Config['Agenda']['pace']
Date = self.Config['Agenda']['begin']
self.Begin = datetime.datetime.strptime(Date.strip(), '%Y-%m-%d %H:%M:%S')
Date = self.Config['Agenda']['end']
self.End = datetime.datetime.strptime(Date.strip(), '%Y-%m-%d %H:%M:%S')
self.CycledAgenda = itertools.cycle(self.Agenda)
class GetHandler:
def __init__(self):
self.DataManager = DataManager()
self.Logger = FileLogger()
def Handle(self, arguments):
try:
self.Logger.Info("GET request received")
parameters = self.Parse(arguments)
return self.GetData(parameters)
except Exception as e:
self.Logger.Error(e)
def Parse(self, arguments):
params = {}
for key in arguments.keys():
params[key] = arguments[key].value
return params
def GetData(self, parameters):
return self.DataManager.Manage(parameters)
def spider_opened(self,spider):
# Get the database parameters from the .cfg file
cfg_parser = CfgFileParser()
self.Config = cfg_parser.file_reader(CONFIGFILE)
# Assign the mongoDB information
self.file_logger = FileLogger()
self.Project = self.Config['Collector']['project']
self.mongo_db = self.Config[self.Project]['mongo_db']
self.Host = self.Config[self.Project]['host']
self.Port = int(self.Config[self.Project]['port'])
self.Collection = self.Config[self.Project]['collection']
self.DataSet = self.Config[self.Project]['dataset']
self.mongo_client = MongoClient(self.Host, self.Port)
self.DBase = self.mongo_client[self.mongo_db]
self.DBase[self.Collection].ensure_index([('idx', pymongo.ASCENDING),('created_at', pymongo.ASCENDING)])
self.Idx = 0
Cursor = self.DBase[self.Collection].find({}).sort([('created_at',pymongo.DESCENDING)]).limit(1)
for Record in Cursor :
self.Idx = Record['idx']
self.file_logger = FileLogger()
self.file_logger.csv_log('PIPELINE', 'open a MONGO pipeline')
class ThesesSpider(scrapy.Spider):
# Defines a spider dedicated to crawling Theses.fr
# works on URL like : http://www.theses.fr/<IDTHESE>
name = 'PhD'
allowed_domains = ["Theses.fr"]
# rules = (Rule(LinkExtractor(allow=(), restrict_xpaths=('//li[@class="pager-next"]',)), callback="parse", follow=True),)
def __init__(self, profile_url='', next=False, *args, **kwargs):
# Get value for given attributes
super(ThesesSpider, self).__init__(*args, **kwargs)
self.start_urls = [profile_url]
self.Next = next
# Instantiate a FileLogger object
self.file_logger = FileLogger()
def parse(self, response):
# Get the page's body HTML
self.file_logger.csv_log('SPIDER', 'path', '')
html_dom = response.xpath('//body').extract()
# Write the HTML DOM into the dom_tree field of a DOM item
dom = DOM(dom_tree=html_dom,
profile_url=self.start_urls[0],
page_url=response.url,
level=1)
self.file_logger.csv_log('SPIDER', 'Success', response.url)
yield dom
def errback(self, failure):
# In case of error, log the error's message
self.file_logger.csv_log('SPIDER', repr(failure), self.start_urls[0])
class PostHandler:
def __init__(self):
self.DBConnection = DBSaver()
self.Logger = FileLogger()
def Handle(self, arguments):
try:
self.Logger.Info("POST request received")
beaconsDict = self.Parse(arguments)
self.Save(beaconsDict)
except Exception as e:
self.Logger.Error(e)
def Parse(self, arguments):
PostObject = arguments.value
self.Logger.Info('Object Recived: ' + PostObject)
josnObject = json.loads(PostObject)
beaconDataDict = josnObject['Beacons']
timestamp = int(time.time())
self.Logger.Info(timestamp)
for beaconData in beaconDataDict:
beaconData["Timestamp"] = timestamp
self.Logger.Info(beaconData)
return beaconDataDict
def Save(self, beaconsDict):
self.Logger.Info("Saving Data")
self.DBConnection.SaveBeaconData(beaconsDict)
class DBRetriever:
def __init__(self):
self.connection = MongoDB('mongodb://localhost:27017/', "mydatabase")
self.Logger = FileLogger()
def QueryDatabase(self, collectionName, filter):
self.Logger.Info('CollectionName: ' + collectionName + ' Query: ' +
str(filter))
cursor = self.connection.QueryCollection(collectionName, filter)
#cursor = self.connection.LoadAllEntries(collectionName)
return self.ParseCursor(cursor)
def ParseCursor(self, cursor):
cursorList = list(cursor)
parsedData = []
for beaconCursor in cursorList:
beaconJSON = dumps(beaconCursor)
beaconObject = json.loads(beaconJSON)
parsedData.append(beaconObject)
self.Logger.Info('ReturnedData: ' + str(parsedData))
return parsedData
def __init__(self, comedy, ip, port=7766):
# call the super init for threading
super(ComedyParser, self).__init__()
# parse comedy
self._comedy = Comedy(comedy)
# init FileLoger
self._logger = FileLogger('log_ComedyParser')
# init laughing and applauding flag
self.laughing = None
self.applauding = None
self.laughter_counter = 0
# Set initial after Response mode to NO.
# This is used to check if the 1sec TTS gap is required
self._afterResponse = False
# init the AudienceAnalyser
self._audienceAnalyser = AudienceAnalyser(expectedPeople,
self._triggerStartFunction,
self._triggerStopFunction)
# start the audience analyser
self._audienceAnalyser.start()
# init communication module
self._communication = CommunicationModule(ip, port)
# connect, auth and start keep-alive thread
self._communication.connect()
# generate all tokens in sentences
self._tokenIterator = self._generateCombinedTokens()
# init response iterator
self._responseTokenIterator = None
def __init__(self, expectedPeople, laughStartCallback, laughStopCallback):
# init FileLoger
self._logger = FileLogger('log_ShoreFrames')
# save the callbacks
self._laughStartCallback = laughStartCallback
self._laughStopCallback = laughStopCallback
# init the Audience
self.audience = Audience(expectedPeople)
# init ReadShore module
self._readShore = ReadShore(self._shoreDataReceived)
# initialise the basic random generator
random.seed()
# last known audience laugh state
self.laughStateLast = 'Not Laughing'
class QueryBuilder:
def __init__(self):
self.Logger = FileLogger()
def Build(self, arguments):
query = self.GetFilter(arguments)
return query
def GetFilter(self, arguments):
self.Logger.Info(arguments)
query = {}
for key in arguments:
if key == 'Offset':
#if (int(arguments[key])) == 0:
#continue
query['Timestamp'] = {
'$gte': (int(time.time()) - (int(arguments[key]) * 60))
}
continue
query[key] = arguments[key]
return query
def __init__(self):
self.DataManager = DataManager()
self.Logger = FileLogger()
altitude = 1.5
yaw = 0
setpoints = []
setpoints += trajectories.takeoff(initial_pos, altitude, yaw)
setpoints += trajectories.xySquare(2.0, altitude, yaw)
setpoints += trajectories.land(initial_pos, altitude, yaw)
"""
# optitrack data for control
ot_position = np.zeros(3)
ot_attitude = np.zeros(3)
cflib.crtp.init_drivers(enable_debug_driver=False)
cf = Crazyflie(rw_cache='./cache')
# logger setup
flogger = FileLogger(cf, cf_uri, fileName)
flogger.enableConfig('otpos')
flogger.enableConfig('otatt')
flogger.enableConfig('attitude')
flogger.enableConfig('gyros')
flogger.enableConfig('acc')
flogger.enableConfig('twr')
flogger.start()
def OT2NED(vector3D_ot):
# convert vector from OT coordinates to NED
vector3D_ned = [0, 0, 0]
vector3D_ned[0] = vector3D_ot[0] # NED.x = OT.x
vector3D_ned[1] = vector3D_ot[2] # NED.y = OT.z
vector3D_ned[2] = -vector3D_ot[1] # NED.z = -OT.y
def __init__(self):
self.DBConnection = DBRetriever()
self.Logger = FileLogger()
self.Locator = LocationCalculator()
self.QueryBuilder = QueryBuilder()
def __init__(self):
self.connection = MongoDB('mongodb://localhost:27017/', "mydatabase")
self.Logger = FileLogger()
def getLogger(self):
if self.isFileLoggingEnabled():
return FileLogger()
else:
return ConsoleLogger()
class AudienceAnalyser:
''' Audience class '''
def __init__(self, expectedPeople, laughStartCallback, laughStopCallback):
# init FileLoger
self._logger = FileLogger('log_ShoreFrames')
# save the callbacks
self._laughStartCallback = laughStartCallback
self._laughStopCallback = laughStopCallback
# init the Audience
self.audience = Audience(expectedPeople)
# init ReadShore module
self._readShore = ReadShore(self._shoreDataReceived)
# initialise the basic random generator
random.seed()
# last known audience laugh state
self.laughStateLast = 'Not Laughing'
def _shoreDataReceived(self, data):
# read the data
self.read(data)
#print data
def start(self):
# start the readShore thread
self._readShore.start()
def stop(self):
pass
def read(self, shoreLine):
# convert shoreLine into dict
line = sp.parseLine(shoreLine)
if ('Frame' in line.keys() and
'Left' in line.keys() and
'Top' in line.keys() and
'Right' in line.keys() and
'Bottom' in line.keys()):
# pass the dict to the Audience object
self.audience.read(line)
# log frame
self._logger.log("Shore frame '%d'" % (line['Frame']))
# determine audience laugh state
if self.laughStateLast == 'Not Laughing':
if self.audience.laughProportion() > 0.333:
self.laughStateLast = 'Laughing'
self._laughStartCallback()
elif self.laughStateLast == 'Laughing':
if self.audience.laughProportion() < 0.111:
self.laughStateLast = 'Not Laughing'
self._laughStopCallback()
class GetCommentTask:
def __init__(self, task, name="extract-comment"):
self.task = task
self.name = name
self.comments = []
self.log = FL('Extracted Comments {:%Y-%m-%d %H:%M:%S}.txt'.format(datetime.datetime.now()))
def init_task(self):
hash_index = 0
loop = True
max_index = len(self.task.hashtags)
next_max_id = ""
while loop:
self.task.check_ops_limit()
current_hash = self.task.hashtags[hash_index]
self.task.api.getHashtagFeed(current_hash, maxid=next_max_id)
print ""
print "CURRENT HASHTAG = " + current_hash
print ""
ig_media = self.task.api.LastJson
if "next_max_id" not in ig_media:
print "####### Changing hashtag #######"
hash_index += 1
next_max_id = ""
if hash_index >= max_index - 1:
break
else:
next_max_id = self.do_task(ig_media)
def do_task(self, ig_media):
last_max_id = ig_media['next_max_id']
if "ranked_items" in ig_media:
key = "ranked_items"
else:
key = "items"
for ig in ig_media[key]:
self.task.api.getMediaComments(ig["id"])
for c in reversed(self.task.api.LastJson['comments']):
txt = c['text']
if self.check_string(txt):
self.comments.append(txt)
print "Comment = " + txt.encode('utf-8', 'ignore').decode('utf-8')
self.log.add_to_file(txt=txt)
self.task.task_count += 1
time.sleep(1)
time.sleep(self.task.get_time_delay())
return last_max_id
"""""
Checks if string doesnt contain special non-english characters, @, or Follow Me.
"""""
def check_string(self,str):
pattern = re.compile("^(?!follow|followme)[\s\w\d\?><;,\{\}\[\]\-_\+=!\#\$%^&\*\|\']*$")
return pattern.match(str)
def __init__(self, task, name="extract-comment"):
self.task = task
self.name = name
self.comments = []
self.log = FL('Extracted Comments {:%Y-%m-%d %H:%M:%S}.txt'.format(datetime.datetime.now()))
def __init__(self):
self.Logger = FileLogger()
class WebPageCollector():
def __init__(self):
cfg_parser = CfgFileParser()
self.Config = cfg_parser.file_reader(CONFIGFILE)
self.Project = self.Config['Collector']['project']
self.Drop = self.Config[self.Project]['drop'] == 'True'
self.Cycle = self.Config[self.Project]['cycle'] == 'True'
self.Next = self.Config[self.Project]['next'] == 'True'
self.mongo_db = self.Config[self.Project]['mongo_db']
self.Host = self.Config[self.Project]['host']
self.Port = int(self.Config[self.Project]['port'])
self.Collection = self.Config[self.Project]['collection']
self.mongo_client = MongoClient(self.Host, self.Port)
self.DBase = self.mongo_client[self.mongo_db]
if self.Drop:
self.DBase[self.Collection].drop()
self.file_logger = FileLogger()
self.set_up()
def set_up(self):
PipeLines = self.Config[self.Project]['pipelines']
PipeLines = json.loads(PipeLines)
self.spider_settings = {
'USER_AGENT':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36 OPR/38.0.2220.41',
'DOWNLOADER_MIDDLEWARES ': {
'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 543
},
'DOWNLOAD_DELAY': 3,
'RANDOMIZE_DOWNLOAD_DELAY': True,
'COOKIES_DEBUG': False,
'COOKIES_ENABLED': True,
'ITEM_PIPELINES': PipeLines,
'LOG_LEVEL': 'INFO'
}
self.SpiderName = self.Config[self.Project]['spider']
self.URLRoller = URLRoller()
if self.Config['Agenda']['pace'] == 'daily':
self.Agenda = DailyAgenda()
if self.Config['Agenda']['pace'] == 'weekly':
self.Agenda = WeeklyAgenda()
self.file_logger.csv_log('COLLECTOR', 'SetUp is done')
def Crawl_job(self, URL, Next):
Runner = CrawlerRunner(settings=self.spider_settings)
return Runner.crawl(eval(self.SpiderName), profile_url=URL, next=Next)
def Schedule_next_crawl(self, null, Now):
self.file_logger.csv_log('COLLECTOR', 'Crawl Session will start')
if self.Agenda.Begin <= Now and Now < self.Agenda.End:
delay = self.Agenda.Delay(Now)
self.file_logger.csv_log('COLLECTOR', 'delayed by', delay)
reactor.callLater(delay, self.LoopCapturing)
else:
self.file_logger.csv_log('COLLECTOR', 'Crawl Session is ended')
reactor.stop()
def Capturing(self):
Now = datetime.datetime.now()
URL = self.URLRoller.next()
print(Now.time(), '->', URL)
Job = self.Crawl_job(URL, self.Next)
Job.addCallback(self.Schedule_next_crawl, Now)
Job.addErrback(self.errback)
def CaptureOnes(self, URL):
print('->', URL)
Now = datetime.datetime.now()
Job = self.Crawl_job(URL, False)
Job.addCallback(self.Schedule_next_crawl, Now)
Job.addErrback(self.errback)
def LoopCapturing(self):
Now = datetime.datetime.now()
URL = self.URLRoller.next()
if URL == self.URLRoller.starter() and not self.Cycle:
reactor.stop()
else:
print(Now.time(), '->', URL)
Job = self.Crawl_job(URL, self.Next)
Job.addCallback(self.Schedule_next_crawl, Now)
Job.addErrback(self.errback)
def errback(self, failure):
print('ERROR:', failure.value)
self.file_logger.csv_log('COLLECTOR', failure.value,
self.start_urls[0])