def get(self):
     try:
         dt =datetime.datetime.now()
         logThis(AEL_LEVEL_INFO, 'Master Task added to its Q at[%s]' %(dt))            
         task_name = GAET_MASTERTASK_NAME + str(datetime.datetime.now())
         task_name = re.sub('[^a-zA-Z0-9_-]', '_', task_name)
         taskqueue.add(queue_name=GAEQ_FOR_MASTERTASK,name=task_name)
         logThis(AEL_LEVEL_INFO, "OK-MASTER TASK ADD")
     except taskqueue.DuplicateTaskNameError:
         logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on QueueAMasterTask-" + traceback.format_exc())
     except:
         logThis(AEL_LEVEL_CRITICAL, "EXP on QueueAMasterTask-" + traceback.format_exc())
Exemplo n.º 2
0
    def get(self):
        try:
            self.response.write('''
    <ul>
      <li><a href="#env">Environment Variables</a></li>
      <li><a href="#filesystem">File System</a></li>
      <li><a href="#request">Request Data</a></li>
    </ul>
    
    <hr noshade><h2 id="env">Environment Variables</h2><table>
    ''')
            keys = os.environ.keys()
            keys.sort()
            for k in keys:
                self.response.write(
                    '<tr><td valign="top"><code>%s</code></td>'
                    '<td valign="top"><code>%s</code></td></tr>' %
                    (cgi.escape(k), cgi.escape(str(os.environ[k]))))
            self.response.write('''
    </table>
    
    <hr noshade><h2 id="filesystem">File System</h2>
    <pre>''')

            for path, dirnames, filenames in os.walk('.'):
                for filename in filenames:
                    self.response.write(
                        cgi.escape(os.path.join(path, filename)) + '\n')
            self.response.write('''
    </pre>
    
    <hr noshade><h2 id="request">Request Data</h2>
    <pre>''')
            self.response.write(cgi.escape(sys.stdin.read()))
            self.response.write('</pre>')

            self.response.write('<hr noshade><p>The time is: %s</p>' %
                                str(datetime.datetime.now()))

        except:
            logThis(AEL_LEVEL_CRITICAL,
                    "EXP on Maina\Page-" + traceback.format_exc())
def fashionista_register_shutdown_hook():
    apiproxy_stub_map.apiproxy.CancelApiCalls()
  
    # We can save state to datastore here or Log some statistics.
    logThis(AEL_LEVEL_WARNING, 'SHUTDOWN IN PROGRESS...')
    logThis(AEL_LEVEL_INFO,'CPU USAGE: %s' % runtime.cpu_usage())
    logThis(AEL_LEVEL_INFO,'MEMORY USAGE: %s' % runtime.memory_usage())
Exemplo n.º 4
0
def fashionista_register_shutdown_hook():
    apiproxy_stub_map.apiproxy.CancelApiCalls()

    # We can save state to datastore here or Log some statistics.
    logThis(AEL_LEVEL_WARNING, 'SHUTDOWN IN PROGRESS...')
    logThis(AEL_LEVEL_INFO, 'CPU USAGE: %s' % runtime.cpu_usage())
    logThis(AEL_LEVEL_INFO, 'MEMORY USAGE: %s' % runtime.memory_usage())
Exemplo n.º 5
0
    def post(self):
        try:
            logThis(AEL_LEVEL_INFO, 'SUBTASK:START')

            Queue = self.request.get('taskQ')

            ac = WindowsCrawler(self.request.get('url'))

            logThis(AEL_LEVEL_INFO, str(ac.isdone()))

            # Retry if task didn't succeed
            if ac.isdone() == False and ac.retry() == True:
                taskName = self.request.get('url')

                task_name_on_Q = '[%s_%s]' % (taskName, datetime.datetime.now(
                ).strftime("[%Y-%m-%d %H:%M:%S]"))
                task_name_on_Q = re.sub('[^a-zA-Z0-9_-]', '_', task_name_on_Q)

                taskqueue.add(queue_name=Queue,
                              name=task_name_on_Q,
                              params={
                                  'url': self.request.get('url'),
                                  'taskQ': Queue
                              })
            elif ac.isdone() == True:
                print "TASK DONE"
                dbA.updateStatus([self.request.get('url')])

        #POST EXCEPT
        except:
            logThis(AEL_LEVEL_CRITICAL,
                    "EXP on HandlerForWindowsTaskQ-" + traceback.format_exc())
    def post(self):
        try:
            logThis(AEL_LEVEL_INFO, 'SUBTASK:START') 
            
            Queue = self.request.get('taskQ')

            print "IOS CRAWL STARTING"
            ac = IOsCrawler(self.request.get('url'))
            
            logThis(AEL_LEVEL_INFO, str(ac.isdone()) )

            # Retry if task didn't succeed
            if ac.isdone() == False and ac.retry() == True:
                taskName = self.request.get('url')
                
                print "THIS IS QUEUE", Queue
                task_name_on_Q = '[%s_%s]' % (taskName, 
                                              datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")) 
                task_name_on_Q = re.sub('[^a-zA-Z0-9_-]', '_', task_name_on_Q)

                taskqueue.add(queue_name=Queue,name=task_name_on_Q,
                              params={'url': self.request.get('url'), 'taskQ' : Queue} )
            elif ac.isdone() == True:
                print "TASK DONE"
                dbA.updateStatus([self.request.get('url')])

        #POST EXCEPT 
        except:
            logThis(AEL_LEVEL_CRITICAL, "EXP on HandlerForAppleTaskQ-" + traceback.format_exc())
class HandlerForMasterTaskQ(webapp2.RequestHandler):
    #HAS TO BE POST FOR QUEUE HANDLERS
    def post(self):
        try:
            logThis(AEL_LEVEL_INFO, 'START PROCESSING MASTERTASK FROM MasterTaskQ')
            #Do Something Here

            dbG.populate('AppStore.GooglePlay_Master_DB')
            urls = dbG.toCrawl()

            for url in urls:
                self._addSubTaskToQueue(url, GAEQ_FOR_GOOGLE)

            dbA.populate('AppStore.AppStore_Master_DB')

            urls = dbA.toCrawl()

            for url in urls:
                self._addSubTaskToQueue(url, GAEQ_FOR_APPLE)

            logThis(AEL_LEVEL_INFO, 'MASTERTASK: Added SubTasks')
        #POST EXCEPT 
        except:
            logThis(AEL_LEVEL_CRITICAL, "EXP on HandlerForMasterTaskQ-" + traceback.format_exc())
    
    def _addSubTaskToQueue(self, taskObj, Queue):
        try:
            if taskObj != None:
                taskName = taskObj
                logThis(AEL_LEVEL_INFO, 'ADDING-SUBTASK [%s] TO SUB TASK Q '% taskName)
                
                task_name_on_Q = '[%s_%s]' % (taskName, 
                                              datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")) 
                task_name_on_Q = re.sub('[^a-zA-Z0-9_-]', '_', task_name_on_Q)
                
                taskqueue.add(queue_name=Queue,name=task_name_on_Q,
                              params={'url': taskObj, 'taskQ' : Queue} )
                logThis(AEL_LEVEL_INFO, "NEW SUB TASK ON Q:" + task_name_on_Q)
                
        except taskqueue.TombstonedTaskError, e:
            logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc())
        except taskqueue.DuplicateTaskNameError, e:
            logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc())
 def _addSubTaskToQueue(self, taskObj, Queue):
     try:
         if taskObj != None:
             taskName = taskObj
             logThis(AEL_LEVEL_INFO, 'ADDING-SUBTASK [%s] TO SUB TASK Q '% taskName)
             
             task_name_on_Q = '[%s_%s]' % (taskName, 
                                           datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")) 
             task_name_on_Q = re.sub('[^a-zA-Z0-9_-]', '_', task_name_on_Q)
             
             taskqueue.add(queue_name=Queue,name=task_name_on_Q,
                           params={'url': taskObj, 'taskQ' : Queue} )
             logThis(AEL_LEVEL_INFO, "NEW SUB TASK ON Q:" + task_name_on_Q)
             
     except taskqueue.TombstonedTaskError, e:
         logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc())
    def post(self):
        try:
            logThis(AEL_LEVEL_INFO, 'START PROCESSING MASTERTASK FROM MasterTaskQ')
            #Do Something Here

            dbG.populate('AppStore.GooglePlay_Master_DB')
            urls = dbG.toCrawl()

            for url in urls:
                self._addSubTaskToQueue(url, GAEQ_FOR_GOOGLE)

            dbA.populate('AppStore.AppStore_Master_DB')

            urls = dbA.toCrawl()

            for url in urls:
                self._addSubTaskToQueue(url, GAEQ_FOR_APPLE)

            logThis(AEL_LEVEL_INFO, 'MASTERTASK: Added SubTasks')
        #POST EXCEPT 
        except:
            logThis(AEL_LEVEL_CRITICAL, "EXP on HandlerForMasterTaskQ-" + traceback.format_exc())
            logThis(AEL_LEVEL_INFO, 'MASTERTASK: Added SubTasks')
        #POST EXCEPT 
        except:
            logThis(AEL_LEVEL_CRITICAL, "EXP on HandlerForMasterTaskQ-" + traceback.format_exc())
    
    def _addSubTaskToQueue(self, taskObj, Queue):
        try:
            if taskObj != None:
                taskName = taskObj
                logThis(AEL_LEVEL_INFO, 'ADDING-SUBTASK [%s] TO SUB TASK Q '% taskName)
                
                task_name_on_Q = '[%s_%s]' % (taskName, 
                                              datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")) 
                task_name_on_Q = re.sub('[^a-zA-Z0-9_-]', '_', task_name_on_Q)
                
                taskqueue.add(queue_name=Queue,name=task_name_on_Q,
                              params={'url': taskObj, 'taskQ' : Queue} )
                logThis(AEL_LEVEL_INFO, "NEW SUB TASK ON Q:" + task_name_on_Q)
                
        except taskqueue.TombstonedTaskError, e:
            logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc())
        except taskqueue.DuplicateTaskNameError, e:
            logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc())
        except:
            logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc())
    
        
                
app = webapp2.WSGIApplication([('/_ah/queue/masterTaskQ', HandlerForMasterTaskQ)
                   ], debug=True)