def execute(self, connector, process_action): job_name = process_action['job_name'] logger = Logger(connector, self) event_repository = EventRepository(connector) process_repository = ProcessRepository(connector) process_property_repository = ProcessPropertyRepository(connector) process_action_property_repository = ProcessActionPropertyRepository( connector) # retrieve the chunk properties process = process_repository.find_by_id(process_action['pcs_id']) event_id = process_property_repository.get_property( process, 'event_id') message = process_property_repository.get_property(process, 'message') backtrace = process_property_repository.get_property( process, 'backtrace') event = event_repository.find_by_id(event_id) state = process_action_property_repository.get_property( process_action, 'state') event['state'] = state event['message'] = message event['backtrace'] = backtrace event_repository.state(event) logger.info(job_name, 'job_name: ' + job_name + " state: " + event['state'])
def execute(self, connector, process_action): job_name = process_action['job_name'] logger = Logger(connector, self) chunk_repository = ChunkRepository(connector) process_repository = ProcessRepository(connector) process_property_repository = ProcessPropertyRepository(connector) process_action_property_repository = ProcessActionPropertyRepository(connector) # retrieve the chunk properties process = process_repository.find_by_id(process_action['pcs_id']) chunk_id = process_property_repository.get_property(process, 'chunk_id') message = process_property_repository.get_property(process, 'message') backtrace = process_property_repository.get_property(process, 'backtrace') chunk = chunk_repository.find_by_id(chunk_id) state = process_action_property_repository.get_property(process_action, 'state') # retrieve the datasource of the payload datasource = chunk['datasource'] payload_ds = DatasourceBuilder.find(connector, datasource) payload_c = ConnectorFactory.create_connector(payload_ds) payload_repository = PayloadRepository(payload_c) payload = json.loads(chunk['payload']) payload_repository.state(chunk, payload, state) payload_c.close() logger.info(job_name, 'job_name: ' + job_name + " state: " + chunk['state'])
def execute(self, connector, process_action): logger = Logger(connector, self) job_name = process_action['job_name'] file_repository = FileRepository(connector) process_repository = ProcessRepository(connector) process_property_repository = ProcessPropertyRepository(connector) process_action_property_repository = ProcessActionPropertyRepository(connector) # retrieve the process properties process = process_repository.find_by_id(process_action['pcs_id']) file_id = process_property_repository.get_property(process, 'file_id') message = process_property_repository.get_property(process, 'message') backtrace = process_property_repository.get_property(process, 'backtrace') file = file_repository.find_by_id(file_id) filename = file['filename'] state = process_action_property_repository.get_property(process_action, 'state') if not state: state = process_property_repository.get_property(process, 'state') if not state: state = 'ERROR_NO_STATE' file['state'] = state file['message'] = message file['backtrace'] = backtrace file_repository.state(file) logger.info(job_name, "filename: " + filename + ", state: " + file['state'])
def execute(self, connector, process_action): logger = Logger(connector, self) job_name = process_action['job_name'] # create instances of classes file_property_repository = FilePropertyRepository(connector) file_repository = FileRepository(connector) process_repository = ProcessRepository(connector) process_property_repository = ProcessPropertyRepository(connector) process_action_property_repository = ProcessActionPropertyRepository( connector) # retrieve the required properties process = process_repository.find_by_id(process_action['pcs_id']) file_id = process_property_repository.get_property(process, 'file_id') name = process_action_property_repository.get_property( process_action, 'name') value = process_action_property_repository.get_property( process_action, 'value') # get the file using the file_id we collected file = file_repository.find_by_id(file_id) filename = file['filename'] file_property_repository.set_property(file, name, value) logger.info( job_name, "filename: " + filename + ", name: " + name + ", value: " + value)
def execute(self, connector, process_action): logger = Logger(connector, self) job_name = process_action['job_name'] process_repository = ProcessRepository(connector) process_property_repository = ProcessPropertyRepository(connector) file_repository = FileRepository(connector) process = process_repository.find_by_id(process_action['pcs_id']) file_id = process_property_repository.get_property(process, 'file_id') file = file_repository.find_by_id(file_id) pickup_location = process_property_repository.get_property( process, 'pickup_location') pickup_filename = file['filename'] pickup_path = process_property_repository.get_property(process, 'path') logger.info( job_name, 'filename: ' + pickup_filename + ", pickup_location: " + pickup_location) # copy the file from pickup folder to the tmp folder. pickup_client = VfsFactory.create_client(pickup_location) pickup_client.connect() pickup_path = pickup_path + os.sep + pickup_filename pickup_client.delete(pickup_path) pickup_client.close() # set the process properties for the next process_action process_property_repository.set_property(process, 'pickup_location', '') process_property_repository.set_property(process, 'path', '')
def execute(self, connector, process_action): logger = Logger(connector, self) job_name = process_action['job_name'] process_repository = ProcessRepository(connector) process_property_repository = ProcessPropertyRepository(connector) process_action_property_repository = ProcessActionPropertyRepository( connector) # retrieve the file properties process = process_repository.find_by_id(process_action['pcs_id']) # retrieve the payload if present payload = JsonExt.loads( process_property_repository.get_property(process, 'payload')) twitter_token = process_action_property_repository.get_property( process_action, 'twitter_token') twitter_token = JsonExt.loads(twitter_token) tweet_content = process_action_property_repository.get_property( process_action, 'tweet_content') tweet_content = ExpressionParser.parse(tweet_content, locals()) tweet_hashtags = process_action_property_repository.get_property( process_action, 'tweet_hashtags') tweet_hashtags = ExpressionParser.parse(tweet_hashtags, locals()) headers = json.loads( process_action_property_repository.get_property( process_action, 'headers')) certificate = JsonExt.loads( process_action_property_repository.get_property( process_action, 'certificate')) proxies = JsonExt.loads( process_action_property_repository.get_property( process_action, 'proxies')) url = process_action_property_repository.get_property( process_action, 'url') message = { 'twitter_token': twitter_token, 'tweet_content': tweet_content, 'tweet_hashtags': tweet_hashtags } logger.info(job_name, json.dumps(message)) response = RestRequest.post(headers, url, message, certificate, proxies) logger.info( job_name, "status_code : " + str(response.status_code) + ", reason : " + response.reason + ", content : " + response.content) if response.status_code != 200: raise ProcessException("status_code : " + str(response.status_code) + ", reason : " + response.reason + ", content : " + response.content)
def execute(self, connector, process_action): logger = Logger(connector, self) job_name = process_action['job_name'] properties = PropertyRepository(connector) process_repository = ProcessRepository(connector) process_property_repository = ProcessPropertyRepository(connector) process_action_property_repository = ProcessActionPropertyRepository( connector) file_repository = FileRepository(connector) process = process_repository.find_by_id(process_action['pcs_id']) file_id = process_property_repository.get_property(process, 'file_id') file = file_repository.find_by_id(file_id) tmp = properties.get_property('scanner.tmp') # retrieve the file properties pickup_location = process_property_repository.get_property( process, 'pickup_location') pickup_filename = file['filename'] pickup_path = process_property_repository.get_property(process, 'path') drop_location = process_action_property_repository.get_property( process_action, 'drop_location') logger.info( job_name, 'filename: ' + pickup_filename + ", pickup_location: " + pickup_location + ", drop_location: " + drop_location) # copy the file from pickup folder to the tmp folder. pickup_client = VfsFactory.create_client(pickup_location) pickup_client.connect() pickup_path = pickup_path + os.sep + pickup_filename tmp_path = tmp + os.sep + pickup_filename pickup_client.get(pickup_path, tmp_path) pickup_client.close() # copy the file form the tmp folder to the drop folder. drop_client = VfsFactory.create_client(drop_location) drop_client.connect() property_path = drop_client.get_path() drop_path = drop_client.get_path() + os.sep + pickup_filename drop_client.put(tmp_path, drop_path) drop_client.close() # set the process properties for the next process_action process_property_repository.set_property(process, 'pickup_location', drop_location) process_property_repository.set_property(process, 'path', property_path)
def execute(self, connector, process_action): job_name = process_action['job_name'] logger = Logger(connector, self) process_action_property_repository = ProcessActionPropertyRepository(connector) # retrieve the properties days = process_action_property_repository.get_property(process_action, 'days') process_property_repository = ProcessPropertyRepository(connector) count = process_property_repository.clean(days) logger.info(job_name, str(count) + ' processes_properties record(s) purged.')
def execute(self, connector, process_action): logger = Logger(connector, self) job_name = process_action['job_name'] file_repository = FileRepository(connector) process_repository = ProcessRepository(connector) process_property_repository = ProcessPropertyRepository(connector) process_action_property_repository = ProcessActionPropertyRepository( connector) # retrieve the file properties process = process_repository.find_by_id(process_action['pcs_id']) payload = process_property_repository.get_property(process, 'payload') if payload: payload = json.loads(payload) datasource = process_action_property_repository.get_property( process_action, 'datasource') method_name = process_action_property_repository.get_property( process_action, 'method_name') params = process_action_property_repository.get_property( process_action, 'params') test_result_params = process_action_property_repository.get_property( process_action, 'test_result_params') if params: params = json.loads(params) params = ExpressionParser.parse(params, locals()) oracle_ds = DatasourceBuilder.find(connector, datasource) oracle_c = ConnectorFactory.create_connector(oracle_ds) scheduler = OracleScheduler(oracle_c) identifier = job_name + '_' + Strings.identifier(10) message = dict() message['method_name'] = method_name message['identifier'] = identifier message['params'] = params if test_result_params: message['test_result_params'] = json.loads(test_result_params) scheduler.create_job(message) logger.info(job_name, json.dumps(message)) process_property_repository.set_property(process, 'identifier', identifier) oracle_c.close()
def execute(self, connector, process_action): logger = Logger(connector, self) job_name = process_action['job_name'] process_repository = ProcessRepository(connector) process_property_repository = ProcessPropertyRepository(connector) process_action_property_repository = ProcessActionPropertyRepository( connector) # retrieve the file properties process = process_repository.find_by_id(process_action['pcs_id']) # retrieve the payload if present payload = process_property_repository.get_property(process, 'payload') if payload: payload = json.loads(payload) datasource = process_action_property_repository.get_property( process_action, 'datasource') method_name = process_action_property_repository.get_property( process_action, 'method_name') params = process_action_property_repository.get_property( process_action, 'params') if params: params = json.loads(params) params = ExpressionParser.parse(params, locals()) oracle_ds = DatasourceBuilder.find(connector, datasource) oracle_c = ConnectorFactory.create_connector(oracle_ds) oracle_call = OracleCall(oracle_c) call = dict() call['method_name'] = method_name call['params'] = params oracle_call.execute(call) logger.info(job_name, json.dumps(call)) oracle_c.close() process_property_repository.set_property(process, 'call', json.dumps(call))
def execute(self, connector, process_action): job_name = process_action['job_name'] logger = Logger(connector, self) file_repository = FileRepository(connector) process_repository = ProcessRepository(connector) process_property_repository = ProcessPropertyRepository(connector) # retrieve the file properties process = process_repository.find_by_id(process_action['pcs_id']) file_id = process_property_repository.get_property(process, 'file_id') file = file_repository.find_by_id(file_id) filename = file['filename'] locked = file_repository.locked(file) if locked: logger.info(job_name, 'file: ' + filename + " locked ") return process_action else: logger.info(job_name, 'file: ' + filename + " not locked ") logger.info(job_name, filename + " state: " + file['state'])
def execute(self, connector, process_action): job_name = process_action['job_name'] logger = Logger(connector, self) properties = PropertyRepository(connector) process_repository = ProcessRepository(connector) process_property_repository = ProcessPropertyRepository(connector) process_action_property_repository = ProcessActionPropertyRepository( connector) file_repository = FileRepository(connector) process = process_repository.find_by_id(process_action['pcs_id']) file_id = process_property_repository.get_property(process, 'file_id') file = file_repository.find_by_id(file_id) tmp = properties.get_property('scanner.tmp') # retrieve the file properties pickup_location = process_property_repository.get_property( process, 'pickup_location') pickup_filename = file['filename'] pickup_path = process_property_repository.get_property(process, 'path') drop_location = process_action_property_repository.get_property( process_action, 'drop_location') # copy the file from pickup folder to the tmp folder. logger.info(job_name, pickup_location + os.sep + pickup_filename) pickup_client = VfsFactory.create_client(pickup_location) pickup_client.connect() pickup_path = pickup_path + os.sep + pickup_filename tmp_path = tmp + os.sep + pickup_filename pickup_client.get(pickup_path, tmp_path) pickup_client.close() # unzip the file and copy the unzipped files to the unzip folder logger.info(job_name, 'unzip ' + pickup_location + os.sep + pickup_filename) zip_client = VfsFactory.create_client("zip://" + tmp_path) zip_client.connect() zip_files = zip_client.list('') for zip_file in zip_files: logger.info(job_name, 'unzipping ' + zip_file['path'] + zip_file['filename']) zip_client.get( zip_file['path'] + zip_file['filename'], tmp + os.sep + zip_file['path'] + zip_file['filename']) # copy the file from the tmp folder to the drop folder logger.info( job_name, 'copying ' + tmp + os.sep + zip_file['path'] + zip_file['filename'] + ' to ' + drop_location) drop_client = VfsFactory.create_client(drop_location) drop_client.connect() tmp_path = zip_file['path'] + zip_file['filename'] drop_path = drop_client.get_path() + os.sep + zip_file['filename'] drop_client.put(tmp_path, drop_path) drop_client.close() logger.info(job_name, pickup_location + os.sep + pickup_filename + " unzipped")
def action(self, connector, message): logger = Logger(self) payload = json.loads(message['payload']) logger.info('', str(payload))
def execute(self, connector, process_action): job_name = process_action['job_name'] logger = Logger(connector, self) process_action_property_repository = ProcessActionPropertyRepository( connector) # retrieve the properties uuid = process_action_property_repository.get_property( process_action, 'uuid') page_size = process_action_property_repository.get_property( process_action, 'page_size') datasource = process_action_property_repository.get_property( process_action, 'datasource') # create the scraper scraper = Clang() response = scraper.mailing_get_quickmails(uuid) resource_id = response.msg # get resource id resource_status = 'BUSY' while resource_status != "READY": response = scraper.resource_get_by_id(uuid, resource_id) resource_status = response.msg.status time.sleep(1) # get mailings resource_size = response.msg.size mailing_ids = [] for i in xrange(0, resource_size): response = scraper.mailing_set_get_mailing_ids( uuid, resource_id, i, 2) mailing_ids.append(response.msg.integer[0]) scraper.resource_free(uuid, resource_id) # get summaries mail_summaries = [] for mailing_id in mailing_ids: response = scraper.mailing_get_by_id(uuid, mailing_id) campaign_name = response.msg.campaignName content_name = response.msg.contentName started_at = response.msg.startedAt ended_at = response.msg.endedAt description = response.msg.description received = response.msg.received unique_clicks = response.msg.uniqueClicks unique_opens = response.msg.uniqueOpens bounces = response.msg.bounces message = dict() message['mailing_id'] = mailing_id message['campaign_name'] = Strings.encode(campaign_name, 'utf-8') message['started_at'] = Strings.encode(started_at, 'utf-8') message['ended_at'] = Strings.encode(ended_at, 'utf-8') message['content_name'] = Strings.encode(content_name, 'utf-8') message['description'] = Strings.encode(description, 'utf-8') message['received'] = received message['unique_clicks'] = unique_clicks message['unique_opens'] = unique_opens message['bounces'] = bounces message['cor'] = MathHelper.divide(unique_opens, received) * 100.0 message['cto'] = MathHelper.divide(unique_clicks, unique_opens) * 100.0 message['ctr'] = MathHelper.divide(unique_clicks, received) * 100.0 mail_summaries.append(message) # delete all the mailings and insert the new ones. mi_ds = DatasourceBuilder.find(datasource) clang_mail_summary_repository = ClangMailSummaryRepository(mi_ds) clang_mail_summary_repository.clean() for summary in mail_summaries: clang_mail_summary_repository.insert(summary) logger.info(job_name, 'job_name: ' + job_name)
def execute(self, connector, process_action): logger = Logger(connector, self) job_name = process_action['job_name'] process_repository = ProcessRepository(connector) process_property_repository = ProcessPropertyRepository(connector) process_action_property_repository = ProcessActionPropertyRepository( connector) print json.dumps(process_action) # retrieve the file properties process = process_repository.find_by_id(process_action['pcs_id']) print json.dumps(process) # retrieve the payload if present payload = JsonExt.loads( process_property_repository.get_property(process, 'payload')) mailjob_id = process_property_repository.get_property( process, 'mailjob_id') auth_info = json.loads( process_action_property_repository.get_property( process_action, 'auth_info')) headers = json.loads( process_action_property_repository.get_property( process_action, 'headers')) certificate = JsonExt.loads( process_action_property_repository.get_property( process_action, 'certificate')) proxies = JsonExt.loads( process_action_property_repository.get_property( process_action, 'proxies')) url = process_action_property_repository.get_property( process_action, 'url') message = {"auth_info": auth_info, "mailjob_id": mailjob_id} logger.info(job_name, json.dumps(message)) response = RestRequest.post(headers, url, message, certificate, proxies) logger.info( job_name, "status_code : " + str(response.status_code) + ", reason : " + response.reason + ", content : " + response.content) if response.status_code != 200: message = "status_code : " + str( response.status_code ) + ", reason : " + response.reason + ", content : " + response.content raise ProcessException(message) content = json.loads(response.content) status = content['status'] numberOfSkipped = content['numberOfSkipped'] if status in ['ENDED_WITH_ERRORS', 'FAILED']: message = "status : " + status + ", reason : " + content['error'] raise ProcessException(message) if numberOfSkipped != 0: message = "numberOfSkipped : " + str( numberOfSkipped ) + ", reason : the email is not send but skipped. Possibly emailadress empty?" raise ProcessException(message) if not status == 'ENDED': return process_action
def execute(self, process_action): job_name = process_action['job_name'] paprika_ds = DatasourceBuilder.build('paprika-ds.json') logger = Logger(self) file_repository = FileRepository(paprika_ds) process_repository = ProcessRepository(paprika_ds) process_property_repository = ProcessPropertyRepository(paprika_ds) process_action_property_repository = ProcessActionPropertyRepository( paprika_ds) # retrieve the file properties process = process_repository.find_by_id(process_action['pcs_id']) file_id = process_property_repository.get_property(process, 'file_id') file = file_repository.find_by_id(file_id) datasource = process_action_property_repository.get_property( process_action, 'datasource') drop_location = process_action_property_repository.get_property( process_action, 'drop_location') filename = drop_location + '/' + file['filename'] CsvFile.normalize(filename, filename + '.norm') source_encoding = CsvFile.guess_encoding(filename + '.norm') CsvFile.iconv(filename + '.norm', source_encoding, filename + '.utf8', 'utf8') delimiter = CsvFile.guess_delimiter(filename + '.utf8') skip_header = False if not header: header = CsvFile.read_header(filename + '.utf8', delimiter) skip_header = True ds = DatasourceBuilder.find(datasource) connector = ConnectorFactory.create_connector(ds) file_repository = FileRepository(ds) file = dict() file['pcs_id'] = 0 file['job_name'] = job_name file['filename'] = filename file['state'] = 'READY' file['rule'] = '' file['hashcode'] = '' file['pickup_location'] = '' file['path'] = '' file['filesize'] = 0 file['pattern'] = '' file = file_repository.insert(file) statics = dict() statics['job_name'] = job_name statics['fle_id'] = file['id'] mapping = 'id.eeid;notification.action;job_name.job_name;fle_id.fle_id' stager = Stager(ds) stager.stage(filename + '.utf8', header, delimiter, 'tripolis_mailings', mapping, skip_header, statics) stager.close() logger.info(job_name, 'job_name: ' + job_name + 'file: ' + filename + " staged")
def run(self, location): abort = self.get_abort() stop = self.get_stop() paprika_ds = DatasourceBuilder.build('paprika-ds.json') connector = ConnectorFactory.create_connector(paprika_ds) logger = Logger(connector, self) job_repository = JobRepository(connector) job = job_repository.job() job_name = job['job_name'] settings = self.get_settings() while self.is_running(): try: properties = PropertyRepository(connector) registry = FileRepository(connector) rule_repository = RuleRepository(connector) excluded_extensions = properties.get_property( 'scanner.excluded_extensions') stable_check_delay = properties.get_property( 'scanner.stable_check_delay') url = location['url'] patterns = location['patterns'] client = VfsFactory.create_client(url) client.set_excluded_extensions(excluded_extensions) client.set_stable_check_delay(int(stable_check_delay)) client.set_regular_expressions(patterns) path = client.get_path() recursive = int(location['recursive']) depth = int(location['depth']) client.connect() files = client.list_stable(path, recursive=recursive, depth=depth) for file in files: registered_file = registry.get_by_hashcode( file['hashcode']) if not registered_file: # find the rule found_rule = None rules = rule_repository.find_by_location(location) for rule in rules: if Matcher.match(ReMethod, rule['pattern'], file['filename']): found_rule = rule if not found_rule: found_rule = rule_repository.find_failsafe() job = job_repository.job() file_job_name = job['job_name'] logger.info( file_job_name, "file: " + file['url'] + '/' + file['filename'] + " rule: " + found_rule['rule'] + " hascode:" + file['hashcode']) process = ProcessService.create_process( connector, found_rule['pdn_id'], file_job_name, found_rule['e_pdn_id']) message = dict() message['job_name'] = file_job_name message['filename'] = file['filename'] message['path'] = file['path'] message['pattern'] = found_rule['pattern'] message['rle_id'] = found_rule['id'] message['rule'] = found_rule['rule'] message['pickup_location'] = file['url'] message['filesize'] = file['size'] message['hashcode'] = file['hashcode'] message['pcs_id'] = process['id'] message['state'] = 'READY' registered_file = registry.insert(message) process_property_repository = ProcessPropertyRepository( connector) process_property_repository.set_property( process, 'file_id', registered_file['id']) process_property_repository.set_property( process, 'pickup_location', file['url']) process_property_repository.set_property( process, 'path', file['path']) process_property_repository.set_property( process, 'payload', json.dumps({ 'filename': file['filename'], 'job_name': file_job_name })) ProcessService.execute_process(connector, process) client.close() # check if we need to abort, can be called from the main thread or other thread aborted = abort.is_aborted() self.running(not aborted) # check if we need to stop, will be set by the agent's WatchWorker thread if not aborted: stopped = stop.is_stopped() self.running(not stopped) connector.close() time.sleep(settings['worker_idle_delay']) logger.trace(job_name, 'worker #' + str(self.get_id()) + " executed.") except: aborted = abort.is_aborted() self.running(not aborted) if not aborted: stopped = stop.is_stopped() self.running(not stopped) result = Traceback.build() logger.fatal(job_name, result['message'], result['backtrace']) connector.close() time.sleep(settings['worker_exception_delay'])