def get(self): try: dt =datetime.datetime.now() logThis(AEL_LEVEL_INFO, 'Master Task added to its Q at[%s]' %(dt)) task_name = GAET_MASTERTASK_NAME + str(datetime.datetime.now()) task_name = re.sub('[^a-zA-Z0-9_-]', '_', task_name) taskqueue.add(queue_name=GAEQ_FOR_MASTERTASK,name=task_name) logThis(AEL_LEVEL_INFO, "OK-MASTER TASK ADD") except taskqueue.DuplicateTaskNameError: logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on QueueAMasterTask-" + traceback.format_exc()) except: logThis(AEL_LEVEL_CRITICAL, "EXP on QueueAMasterTask-" + traceback.format_exc())
def get(self): try: self.response.write(''' <ul> <li><a href="#env">Environment Variables</a></li> <li><a href="#filesystem">File System</a></li> <li><a href="#request">Request Data</a></li> </ul> <hr noshade><h2 id="env">Environment Variables</h2><table> ''') keys = os.environ.keys() keys.sort() for k in keys: self.response.write( '<tr><td valign="top"><code>%s</code></td>' '<td valign="top"><code>%s</code></td></tr>' % (cgi.escape(k), cgi.escape(str(os.environ[k])))) self.response.write(''' </table> <hr noshade><h2 id="filesystem">File System</h2> <pre>''') for path, dirnames, filenames in os.walk('.'): for filename in filenames: self.response.write( cgi.escape(os.path.join(path, filename)) + '\n') self.response.write(''' </pre> <hr noshade><h2 id="request">Request Data</h2> <pre>''') self.response.write(cgi.escape(sys.stdin.read())) self.response.write('</pre>') self.response.write('<hr noshade><p>The time is: %s</p>' % str(datetime.datetime.now())) except: logThis(AEL_LEVEL_CRITICAL, "EXP on Maina\Page-" + traceback.format_exc())
def fashionista_register_shutdown_hook(): apiproxy_stub_map.apiproxy.CancelApiCalls() # We can save state to datastore here or Log some statistics. logThis(AEL_LEVEL_WARNING, 'SHUTDOWN IN PROGRESS...') logThis(AEL_LEVEL_INFO,'CPU USAGE: %s' % runtime.cpu_usage()) logThis(AEL_LEVEL_INFO,'MEMORY USAGE: %s' % runtime.memory_usage())
def fashionista_register_shutdown_hook(): apiproxy_stub_map.apiproxy.CancelApiCalls() # We can save state to datastore here or Log some statistics. logThis(AEL_LEVEL_WARNING, 'SHUTDOWN IN PROGRESS...') logThis(AEL_LEVEL_INFO, 'CPU USAGE: %s' % runtime.cpu_usage()) logThis(AEL_LEVEL_INFO, 'MEMORY USAGE: %s' % runtime.memory_usage())
def post(self): try: logThis(AEL_LEVEL_INFO, 'SUBTASK:START') Queue = self.request.get('taskQ') ac = WindowsCrawler(self.request.get('url')) logThis(AEL_LEVEL_INFO, str(ac.isdone())) # Retry if task didn't succeed if ac.isdone() == False and ac.retry() == True: taskName = self.request.get('url') task_name_on_Q = '[%s_%s]' % (taskName, datetime.datetime.now( ).strftime("[%Y-%m-%d %H:%M:%S]")) task_name_on_Q = re.sub('[^a-zA-Z0-9_-]', '_', task_name_on_Q) taskqueue.add(queue_name=Queue, name=task_name_on_Q, params={ 'url': self.request.get('url'), 'taskQ': Queue }) elif ac.isdone() == True: print "TASK DONE" dbA.updateStatus([self.request.get('url')]) #POST EXCEPT except: logThis(AEL_LEVEL_CRITICAL, "EXP on HandlerForWindowsTaskQ-" + traceback.format_exc())
def post(self): try: logThis(AEL_LEVEL_INFO, 'SUBTASK:START') Queue = self.request.get('taskQ') print "IOS CRAWL STARTING" ac = IOsCrawler(self.request.get('url')) logThis(AEL_LEVEL_INFO, str(ac.isdone()) ) # Retry if task didn't succeed if ac.isdone() == False and ac.retry() == True: taskName = self.request.get('url') print "THIS IS QUEUE", Queue task_name_on_Q = '[%s_%s]' % (taskName, datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")) task_name_on_Q = re.sub('[^a-zA-Z0-9_-]', '_', task_name_on_Q) taskqueue.add(queue_name=Queue,name=task_name_on_Q, params={'url': self.request.get('url'), 'taskQ' : Queue} ) elif ac.isdone() == True: print "TASK DONE" dbA.updateStatus([self.request.get('url')]) #POST EXCEPT except: logThis(AEL_LEVEL_CRITICAL, "EXP on HandlerForAppleTaskQ-" + traceback.format_exc())
class HandlerForMasterTaskQ(webapp2.RequestHandler): #HAS TO BE POST FOR QUEUE HANDLERS def post(self): try: logThis(AEL_LEVEL_INFO, 'START PROCESSING MASTERTASK FROM MasterTaskQ') #Do Something Here dbG.populate('AppStore.GooglePlay_Master_DB') urls = dbG.toCrawl() for url in urls: self._addSubTaskToQueue(url, GAEQ_FOR_GOOGLE) dbA.populate('AppStore.AppStore_Master_DB') urls = dbA.toCrawl() for url in urls: self._addSubTaskToQueue(url, GAEQ_FOR_APPLE) logThis(AEL_LEVEL_INFO, 'MASTERTASK: Added SubTasks') #POST EXCEPT except: logThis(AEL_LEVEL_CRITICAL, "EXP on HandlerForMasterTaskQ-" + traceback.format_exc()) def _addSubTaskToQueue(self, taskObj, Queue): try: if taskObj != None: taskName = taskObj logThis(AEL_LEVEL_INFO, 'ADDING-SUBTASK [%s] TO SUB TASK Q '% taskName) task_name_on_Q = '[%s_%s]' % (taskName, datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")) task_name_on_Q = re.sub('[^a-zA-Z0-9_-]', '_', task_name_on_Q) taskqueue.add(queue_name=Queue,name=task_name_on_Q, params={'url': taskObj, 'taskQ' : Queue} ) logThis(AEL_LEVEL_INFO, "NEW SUB TASK ON Q:" + task_name_on_Q) except taskqueue.TombstonedTaskError, e: logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc()) except taskqueue.DuplicateTaskNameError, e: logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc())
def _addSubTaskToQueue(self, taskObj, Queue): try: if taskObj != None: taskName = taskObj logThis(AEL_LEVEL_INFO, 'ADDING-SUBTASK [%s] TO SUB TASK Q '% taskName) task_name_on_Q = '[%s_%s]' % (taskName, datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")) task_name_on_Q = re.sub('[^a-zA-Z0-9_-]', '_', task_name_on_Q) taskqueue.add(queue_name=Queue,name=task_name_on_Q, params={'url': taskObj, 'taskQ' : Queue} ) logThis(AEL_LEVEL_INFO, "NEW SUB TASK ON Q:" + task_name_on_Q) except taskqueue.TombstonedTaskError, e: logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc())
def post(self): try: logThis(AEL_LEVEL_INFO, 'START PROCESSING MASTERTASK FROM MasterTaskQ') #Do Something Here dbG.populate('AppStore.GooglePlay_Master_DB') urls = dbG.toCrawl() for url in urls: self._addSubTaskToQueue(url, GAEQ_FOR_GOOGLE) dbA.populate('AppStore.AppStore_Master_DB') urls = dbA.toCrawl() for url in urls: self._addSubTaskToQueue(url, GAEQ_FOR_APPLE) logThis(AEL_LEVEL_INFO, 'MASTERTASK: Added SubTasks') #POST EXCEPT except: logThis(AEL_LEVEL_CRITICAL, "EXP on HandlerForMasterTaskQ-" + traceback.format_exc())
logThis(AEL_LEVEL_INFO, 'MASTERTASK: Added SubTasks') #POST EXCEPT except: logThis(AEL_LEVEL_CRITICAL, "EXP on HandlerForMasterTaskQ-" + traceback.format_exc()) def _addSubTaskToQueue(self, taskObj, Queue): try: if taskObj != None: taskName = taskObj logThis(AEL_LEVEL_INFO, 'ADDING-SUBTASK [%s] TO SUB TASK Q '% taskName) task_name_on_Q = '[%s_%s]' % (taskName, datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")) task_name_on_Q = re.sub('[^a-zA-Z0-9_-]', '_', task_name_on_Q) taskqueue.add(queue_name=Queue,name=task_name_on_Q, params={'url': taskObj, 'taskQ' : Queue} ) logThis(AEL_LEVEL_INFO, "NEW SUB TASK ON Q:" + task_name_on_Q) except taskqueue.TombstonedTaskError, e: logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc()) except taskqueue.DuplicateTaskNameError, e: logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc()) except: logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc()) app = webapp2.WSGIApplication([('/_ah/queue/masterTaskQ', HandlerForMasterTaskQ) ], debug=True)