def __getstate__(self): """ For pickling the report when passing it to Celery. """ logging = get_task_logger(__name__) # logging is likely to happen within celery. # pickle only what the report needs from the request object request = dict( GET=self.request.GET, META=dict( QUERY_STRING=self.request.META.get('QUERY_STRING'), PATH_INFO=self.request.META.get('PATH_INFO') ), datespan=self.request.datespan, couch_user=None ) try: request.update(couch_user=self.request.couch_user.get_id) except Exception as e: logging.error("Could not pickle the couch_user id from the request object for report %s. Error: %s" % (self.name, e)) return dict( request=request, request_params=self.request_params, domain=self.domain, context={} )
def run(self, plugin_cls_name, settings, logger=None): """ Just load the right plugin class, and then execute the check. Return tuple of (RESULT, MSG). MSG is empty for UP states. """ plugin = get_cls_by_name(plugin_cls_name)() plugin.set_logger(logger or get_task_logger('django')) msg = None try: msg = plugin.run_check(settings) except PluginImplementationError as e: return (Service.STATE_UNKNOWN, e.message) except PluginConfigurationError as e: return (Service.STATE_UNKNOWN, "Plugin {} is misconfigured: {}".format( plugin_cls_name, e.message)) except ServiceCheckFailed as e: return (Service.STATE_UNKNOWN, "Service check failed: " + e.reason) except ServiceIsDown as e: return (Service.STATE_DOWN, e.message) except ServiceHasWarning as e: return (Service.STATE_WARNING, e.message) except Exception as e: return (Service.STATE_UNKNOWN, 'Unknown exception: {}: {}'.format( e.__class__, e)) return (Service.STATE_OK, msg or '')
def __setstate__(self, state): """ For unpickling a pickled report. """ logging = get_task_logger(__name__) # logging lis likely to happen within celery. self.domain = state.get('domain') self.context = state.get('context', {}) class FakeHttpRequest(object): GET = {} META = {} couch_user = None datespan = None request_data = state.get('request') request = FakeHttpRequest() request.GET = request_data.get('GET', {}) request.META = request_data.get('META', {}) request.datespan = request_data.get('datespan') try: couch_user = CouchUser.get(request_data.get('couch_user')) request.couch_user = couch_user except Exception as e: logging.error("Could not unpickle couch_user from request for report %s. Error: %s" % (self.name, e)) self.request = request self._caching = True self.request_params = state.get('request_params') self._update_initial_context()
def fetch_classes(page, object_, identity): logger = get_task_logger(__name__ + '.fetch_classes') session = get_session() res = select_by_class( s=identity, s_name='name', entities=object_.PROPERTIES, page=page, p=object_.TYPE_PREDICATES, ) current_time = datetime.datetime.now(datetime.timezone.utc) logger.warning('fetching %s, %d', identity, len(res)) for item in res: try: with session.begin(): new_entity = object_.initialize(item) new_entity.last_crawled = current_time new_entity = session.merge(new_entity) session.add(new_entity) except IntegrityError: entities = session.query(object_) \ .filter_by( name=item['name'] ) if entities.count() > 0: entity = entities.one() entity.last_crawled = current_time entity.initalize(item)
def delete_order_from_users_google_calendar(order_id, target_users=None): from orders_manager.models import User, Order from orders_manager.google_apis import GoogleApiHandler logger = get_task_logger(__name__) google_api_handler = GoogleApiHandler() order = Order.objects.get(id=order_id) if target_users is None: target_users = set([i.user_id for i in order.program_executors.all()]) target_users.update( [i.executor.user_id for i in order.additional_services_executors.all()] ) results = {} try: for user_id in target_users: user = User.objects.get(id=user_id) res = google_api_handler.delete_event_from_user_calendar( user, order.hex_id()) results.update({user.get_full_name(): res}) send_order_notice_to_email(order, user, action_type='delete') except Exception as ex: logger.error(ex.args[0]) return ex.args[0] return results
def __init__(self, ps_client, es_client, array_context): self._ps_client = ps_client self._es_client = es_client self._array_name = array_context.name self._array_id = array_context.id self._data_ttl = array_context.data_ttl self.logger = get_task_logger(__name__)
def fetch_link(url, session, *, log_prefix=''): '''Returns result, tree, namespace, name, final_url.''' logger = get_task_logger(__name__ + '.fetch_link') if not is_wiki_page(url): return False, None, None, None, url r = requests.get(url) try: final_url = r.url[:r.url.index('?')] except ValueError: final_url = r.url if not is_wiki_page(final_url): return False, None, None, None, final_url tree = document_fromstring(r.text) try: namespace = tree.xpath('//div[@class="pagetitle"]')[0] \ .text.strip()[:-1] except (AttributeError, AssertionError, IndexError): logger.warning('%sWarning on url %s: ' 'There is no pagetitle on this page. Ignoring.', log_prefix, url) return False, tree, None, None, final_url if namespace == '': namespace = 'Main' name = tree.xpath('//div[@class="pagetitle"]/span')[0].text.strip() type = determine_type(namespace) if type == 'Administrivia': return False, tree, namespace, name, final_url upsert_entity(session, namespace, name, type, final_url) process_redirections(session, url, final_url, namespace, name) return True, tree, namespace, name, final_url
def publish_ready_projects(): ready_projects_to_publish = api.models.Project.objects.filter( publish_mode=api.models.Project.PUBLISH_MODE_READY, min_publish_date__lte=utc_now(), ) published_projects_ids = [] for project in ready_projects_to_publish: old_publish_mode = project.publish_mode project.publish_mode = api.models.Project.PUBLISH_MODE_PUBLISHED project.save() published_projects_ids.append(project.id) project.notify_owner( 'project_publish_mode_change_by_target', { 'target': None, 'description': 'Project "%s" published because publication date has arrived - %s.' %(project.title, project.min_publish_date.strftime('%Y-%m-%d %H:%M')), 'publishMode': project.publish_mode, 'oldPublishMode': old_publish_mode, 'publishDate': project.publish_date.strftime('%Y-%m-%d %H:%M'), }, send_mail_with_template='IGNITE_notification_publish_mode_change', ) if published_projects_ids: logger = get_task_logger('publish_ready_projects') logger.info('Published %i projects that were ready: %s.', len(published_projects_ids), ', '.join([str(x) for x in published_projects_ids]))
def archive(job_pk): """Starts a celery.chord that runs stat_addon for each complete addon attached to the Node, then runs #archive_node with the result :param job_pk: primary key of ArchiveJob :return: None """ create_app_context() job = ArchiveJob.load(job_pk) src, dst, user = job.info() logger = get_task_logger(__name__) logger.info("Received archive task for Node: {0} into Node: {1}".format(src._id, dst._id)) return celery.chain( [ celery.group( stat_addon.si( addon_short_name=target.name, job_pk=job_pk, ) for target in job.target_addons ), archive_node.s( job_pk=job_pk ) ] )
def getdiskusage(directory): # Try an all-python solution here - in case the suprocess spawning is causing grief. We could be opening # hundreds of instances of shells above. logger = get_task_logger('iondb.rundb.tasks.getdiskusage') def dir_size (start): if not start or not os.path.exists(start): return 0 file_walker = ( os.path.join(root, f) for root, _, files in os.walk( start ) for f in files ) total = 0L for f in file_walker: if os.path.isdir(f): total += dir_size(f) continue if not os.path.isfile(f): continue try: total += os.lstat(f).st_size except OSError: logger.exception("Cannot stat %s during calc_size", f) return total # Returns size in MB return dir_size(directory)/(1024*1024)
def close_spider(self, spider): """Post results to BROKER API.""" from celery.utils.log import get_task_logger logger = get_task_logger(__name__) if 'SCRAPY_JOB' not in os.environ: self._cleanup(spider) return if hasattr(spider, 'tmp_dir'): shutil.rmtree(path=spider.tmp_dir, ignore_errors=True) errors = getattr(spider, 'state', {}).get('errors', []) if self.count > 0 or errors: task_endpoint = spider.settings[ 'API_PIPELINE_TASK_ENDPOINT_MAPPING' ].get( spider.name, spider.settings['API_PIPELINE_TASK_ENDPOINT_DEFAULT'], ) logger.info('Triggering celery task: %s.', task_endpoint) kwargs = self._prepare_payload(spider) logger.debug( ' Sending results:\n %s', pprint.pformat(kwargs), ) res = self.celery.send_task(task_endpoint, kwargs=kwargs) logger.info('Sent celery task %s', res) self._cleanup(spider)
def create_celery_app(flask_app=None): logger.debug("Creating celery app") if not flask_app: if has_app_context(): logger.debug("Using current flask app") app = current_app else: logger.debug("No current flask app") app = create_app() else: app = flask_app celery = Celery(app.import_name, broker=app.config['CELERY_BROKER_URL'], backend=app.config['CELERY_RESULT_BACKEND']) celery.conf.update(app.config) TaskBase = celery.Task class ContextTask(TaskBase): abstract = True def __call__(self, *args, **kwargs): with app.app_context(): return TaskBase.__call__(self, *args, **kwargs) celery.Task = ContextTask celery.logger = get_task_logger(__name__) app.celery = celery return app
def send_order_to_users_google_calendar( order_id, send_email=True, is_full_description=None, is_new_order=False): from orders_manager.models import Order, UserProfile from orders_manager.google_apis import GoogleApiHandler logger = get_task_logger(__name__) google_api_handler = GoogleApiHandler() order = Order.objects.get(id=order_id) if is_full_description is None: dt = datetime.datetime.strptime( '%s %s' % (order.celebrate_date, '00:00:00'), '%Y-%m-%d %H:%M:%S') today = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) dt_lim = today + datetime.timedelta(2) is_full_description = 0 < (dt_lim - dt).days <= 2 date_str = '{0} {1}'.format(order.celebrate_date, order.celebrate_time) event_start = datetime.datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S') event_duration = int(order.duration) event_end = event_start + datetime.timedelta(0, event_duration * 60) event_start = event_start.isoformat() event_end = event_end.isoformat() logger.debug('Sending order \'%s\' start \'%s\'' % (order.program.title, event_start)) executor_to_event_title = {} description = _get_order_description(order, is_full_description) for program_executor in order.program_executors.all(): executor_to_event_title[program_executor.user_id] = order.program.title for service_to_executors in order.additional_services_executors.all(): if executor_to_event_title.get(service_to_executors.executor.user_id): executor_to_event_title[service_to_executors.executor.user_id] += \ ' + %s' % service_to_executors.additional_service.title else: executor_to_event_title[service_to_executors.executor.user_id] = \ service_to_executors.additional_service.title for user_id, title in executor_to_event_title.items(): executor = UserProfile.objects.get(user_id=user_id) try: summary = _get_order_summary(order, is_full_description) summary = summary.format(title=title) google_api_handler.send_event_to_user_calendar( executor, order.hex_id(), event_start, event_end, summary, description) if send_email is True: send_order_notice_to_email(order, executor, 'create' if is_new_order else 'update') except Exception as ex: logger.error(ex.args[0]) return '{0} was updated.'.format(order.program.title)
def _apply_matrix(self, s, m): """Apply the matrix.""" log = get_task_logger(__name__) s.shape = (1024, -1) coeffs = m.dot(s) coeffs = coeffs.view(np.ndarray) return coeffs
def select_dbpedia(query): logger = get_task_logger(__name__ + '.select_dbpedia') sparql = SPARQLWrapper("http://dbpedia.org/sparql") sparql.setReturnFormat(JSON) sparql.setQuery('''PREFIX dbpedia-owl: <http://dbpedia.org/ontology/> PREFIX dbpprop: <http://dbpedia.org/property/>'''+query) tried = 0 wikipedia_limit = get_wikipedia_limit() while tried < wikipedia_limit: try: tried = tried + 1 tuples = sparql.query().convert()['results']['bindings'] except HTTPError as e: logger.exception('HTTPError %s: %s, tried %d/%d', e.code, e.reason, tried, wikipedia_limit) except URLError as e: logger.exception('URLError %s, tried %d/%d', e.args, tried, wikipedia_limit) except ConnectionResetError as e: logger.exception('ConnectionResetError %s', e) except IncompleteRead as e: logger.exception('Network Error, retry %d', tried) except EndPointNotFound as e: logger.exception('SQLAlchemy Error, retry %d', tried) else: return[{k: v['value'] for k, v in tupl.items()} for tupl in tuples] return []
def dispatch_task(task, log_name, r): logger = get_task_logger(log_name) task_id = task[0] url_path = task[1] max_depth = task[2] allow_domains = task[3] urls = [] #check the argument if task_id == None: logger.error('task_id is None.') return if url_path == None: logger.error('url_path is None.') return if max_depth == None: logger.error('max_depth is None.') return elif max_depth < 0: logger.debug('max depth less than ZERO, set it to ZERO.') max_depth = 0 logger.info(task) logger.info(task[1]) try: with open(url_path, 'r') as f: urls = f.readlines() except Exception, e: logger.error(e) return
def errLog(x): """ Logs x to celery INFO. Used as a callback in sh piping to manually print otherwise swallowed error logs. """ logger = get_task_logger(__name__) logger.info(x)
def save_link(name, url): with psycopg2.connect(worker.conf.DB_FILENAME) as conn, \ conn.cursor() as cur: try: cur.execute('INSERT INTO entities VALUES (%s, %s, %s, NULL, NULL)', (name[0], name[1], url)) except psycopg2.IntegrityError: cur.execute('UPDATE entities SET url = %s ' 'WHERE namespace = %s and name = %s', (url, name[0], name[1])) conn.rollback() conn.commit() cur.execute('SELECT count(*) FROM entities') get_task_logger(__name__ + '.save_link').info( 'Total %d', cur.fetchone()[0] )
def get_logger(name): try: from celery.utils.log import get_task_logger logger = get_task_logger(name) except ImportError: import logging logger = logging.getLogger(name) return logger
def facebook_update(): log = get_task_logger('facebook') log.info('[Facebook] Start') fbapi = FacebookAPI( settings.SOCIAL_FACEBOOK_APP_ID, settings.SOCIAL_FACEBOOK_APP_SECRET) fb = FacebookUpdater(fbapi) fb.update() log.info('[Facebook] End')
def printDb(self, id): ''' Useful task for probing the boxm2 database of a worker ''' import boxm2_batch from vsi.tools import Redirect, Logger from celery.utils.log import get_task_logger l = get_task_logger(__name__); with Redirect(all=Logger(l)): l.error(boxm2_batch.print_db())
def logger(self): """ Get the logger for this action. """ logname = self.__class__.get_name() if self.executed_by_celery: return get_task_logger(logname) else: return logging.getLogger(logname)
def get_redis_logger(id_): loggy = get_task_logger(id_) loggy.setLevel(logging.DEBUG) loggy.propagate = False loggy.handlers = [] loggy.addHandler(RedisHandler(id_)) # for test purposes: # loggy.addHandler(logging.StreamHandler()) return loggy
def run(self, *args, **kwargs): if get_task_logger: self.logger = get_task_logger(self.__class__.__name__) else: # Celery 2.X fallback self.logger = self.get_logger(**kwargs) self.logger.info("Starting %s", self.__class__.__name__) self.cache_key = self._get_cache_key(**kwargs) # Record start time to give estimated time remaining estimates self.start_time = time.time() # Keep track of progress updates for update_frequency tracking self._last_update_count = 0 # Report to the backend that work has been started. if self.request.id: self.update_state(None, PROGRESS, { "progress_percent": 0, "time_remaining": -1, }) memleak_threshold = int(getattr(self, 'memleak_threshold', -1)) if memleak_threshold >= 0: begining_memory_usage = self._get_memory_usage() self.logger.info("Calculating result") try: task_result = self.calculate_result(*args, **kwargs) except Exception: # Don't want other tasks waiting for this task to finish, since it # won't self._break_thundering_herd_cache() raise # We can use normal celery exception handling for this if hasattr(self, 'cache_duration'): cache_duration = self.cache_duration else: cache_duration = -1 # By default, don't cache if cache_duration >= 0: # If we're configured to cache this result, do so. cache.set(self.cache_key, self.request.id, cache_duration) # Now that the task is finished, we can stop all of the thundering herd # avoidance self._break_thundering_herd_cache() if memleak_threshold >= 0: self._warn_if_leaking_memory( begining_memory_usage, self._get_memory_usage(), memleak_threshold, task_kwargs=kwargs, ) return task_result
def process_action(game_id, lsn, replay=False): logger = get_task_logger(game_id) action_queue = [] with transaction.atomic(): g = Game.objects.get(hashkey=game_id) logs = GameLog.objects.filter(game=g, lsn__gt=g.applied_lsn, lsn__lte=lsn, status=GameLog.ACCEPTED) info = json.loads(g.current_info, cls=GameInfoDecoder) for l in logs: action_dict = l.get_log_as_dict() user_id = l.player.user_id if l.player != None else None try: state = GameState.getInstance(info) logger.info('Applying ' + str(l) + ': ' + type(state).__name__ + ', ' + l.log) result = state.action(action_dict['action'], user_id=user_id, params=action_dict) info = state.info if 'random' in result.keys(): action_dict['random'] = result['random'] if 'queue_action' in result.keys() : action_queue.append(result['queue_action']) if 'msg' in result.keys(): for m in result['msg']: l.add_info(m['user_id'], m['msg']) l.set_log(action_dict) l.status = GameLog.CONFIRMED except (GameState.NotSupportedAction, GameState.InvalidAction, Action.InvalidParameter, Action.WarNotResolved) as e: l.status = GameLog.FAILED logger.error(str(l) + " :" + type(e).__name__ + ": " + e.message) l.add_warning(user_id, e.message) g.applied_lsn = l.lsn l.save() g.set_current_info(info) if replay == False: for aq in action_queue: g.last_lsn += 1 p = Player.objects.get(user_id=aq['_player']) if '_player' in aq else None a = GameLog( game=g, player=p, lsn=g.last_lsn, ) a.set_log(log_dict=aq) a.save() g.save() if action_queue and replay == False: process_action.delay(g.hashkey, g.last_lsn) return g.applied_lsn
def open(self): """Open the dataset, as a context manager.""" log = get_task_logger(__name__) with self.dataset.open() as g: try: yield g[self.kind.h5path] except KeyError as e: log.error("Error opening {0} from {1} in {2}".format(self.kind.h5path, g.name, repr(self.dataset))) log.error("File: {0}".format(self.dataset.filename)) raise
def query_transactions(): logger = get_task_logger('bitcoin_transactions') logger.info("query_transactions: start task") with NonBlockingCacheLock("query_transactions_ongoing"): try: monitor_transactions(logger) except Exception as e: logger.error("query_transactions: exception %s\n%s" % (e, format_exc(e))) return
def refresh_all_wechat_token(self): """ 定时1小时,刷新所有已授权公众号 """ logger = get_task_logger('refresh_all_wechat_token') for wechat in Wechat.objects.exclude(appid=settings.TEST_APPID).all(): if not wechat.authorized: logger.error('公众号{0}失去授权'.format(wechat.appid)) continue refresh_wechat_token.delay(wechat.appid)
def process_wechat_query_auth_code_test(FromUserName, query_auth_code): """ 处理发布前微信的自动化测试query_auth_code """ logger = get_task_logger('process_wechat_query_auth_code_test') logger.info(FromUserName) logger.info(query_auth_code) component = get_component() client = component.get_client_by_authorization_code(query_auth_code) client.message.send_text(FromUserName, query_auth_code+'_from_api')
def add_numbers(self, row_id): logger = get_task_logger(__name__) logger.info(u'[{0.id}]Function all_numbers called with params [{1}] with extended info:{0}'.format( self.request, row_id )) from .models import Adder record = Adder.objects.get(pk=row_id) record.result = record.x + record.y record.save()
# -*- coding: utf-8 -*- from __future__ import absolute_import import pickle import pandas as pd from handler.iddb_handler import TwseIdDBHandler, OtcIdDBHandler from handler.hisdb_handler import TwseHisDBHandler, OtcHisDBHandler from giant.celery import app from celery import shared_task from celery.utils.log import get_task_logger logger = get_task_logger('handler') hisdb_tasks = {'twse': TwseHisDBHandler, 'otc': OtcHisDBHandler} iddb_tasks = {'twse': TwseIdDBHandler, 'otc': OtcIdDBHandler} hisitems = ['stock', 'credit', 'future', 'trader'] iditems = ['stock', 'trader'] @shared_task(time_limit=60 * 60) def collect_iditem(stream): args, kwargs = pickle.loads(stream) opt = kwargs.pop('opt', None) targets = kwargs.pop('targets', []) callback = kwargs.pop('callback', None) debug = kwargs.pop('debug', False)
from pytz import UTC from xmodule.modulestore.django import modulestore from track.views import task_track from courseware.grades import iterate_grades_for from courseware.models import StudentModule from courseware.model_data import FieldDataCache from courseware.module_render import get_module_for_descriptor_internal from instructor_analytics.basic import enrolled_students_features from instructor_analytics.csvs import format_dictlist from instructor_task.models import ReportStore, InstructorTask, PROGRESS from student.models import CourseEnrollment # define different loggers for use within tasks and on client side TASK_LOG = get_task_logger(__name__) # define value to use when no task_id is provided: UNKNOWN_TASK_ID = 'unknown-task_id' # define values for update functions to use to return status to perform_module_state_update UPDATE_STATUS_SUCCEEDED = 'succeeded' UPDATE_STATUS_FAILED = 'failed' UPDATE_STATUS_SKIPPED = 'skipped' class BaseInstructorTask(Task): """ Base task class for use with InstructorTask models. Permits updating information about task in corresponding InstructorTask for monitoring purposes.
import datetime import requests import pandas as pd from utilities.models import db_config import math import psycopg2 import time from celery.utils.log import get_task_logger from metar.Metar import Metar, ParserError logger = get_task_logger("redemet") def get_date_and_standard_metar(raw_data): date_str, partially_cleaned_data = raw_data.split(' - ') observation_time = datetime.datetime.strptime(date_str, '%Y%m%d%H') # The default Metar expects COR modifiers to come after the # time data. We will just remove the COR reference and let it # be parsed as a regular entry (since it makes no difference # for our purposes). partially_cleaned_data = partially_cleaned_data.replace('COR ', '') cleaned_data = partially_cleaned_data.rstrip('=') return observation_time, cleaned_data def humidity(temperature, dew_point): term_a = (17.625 * dew_point) / (243.04 + dew_point) term_b = (17.625 * temperature) / (243.04 + temperature) return 100 * (math.exp(term_a) / math.exp(term_b))
from datetime import datetime from datetime import timedelta from influxdb import InfluxDBClient from .. import curse_login from .. import celery from .. import db from .. import app from .. import redis_store from ..models import AddonModel, AddonStatusEnum from .task_helpers import request_addons from .task_helpers import request_addons_by_id from .task_helpers import request_all_files logger = get_task_logger(__name__) @celery.task def p_curse_checklogin(): return curse_login.renew_session() @celery.task def p_remove_expired_caches(): return requests_cache.core.remove_expired_responses() @celery.task def p_fill_incomplete_addons(): request_addons(AddonModel.query.filter(AddonModel.name == None).all())
def logtask(): logger = get_task_logger(__name__) while getattr(logger, 'parent', None): assert logger.propagate == 1 logger = logger.parent
"""Plugin implementation for a simple web application.""" from celery.utils.log import get_task_logger from cloudlaunch.backend_plugins.base_vm_app import BaseVMAppPlugin from cloudlaunch.backend_plugins.cloudman2_app import get_iam_handler_for from cloudlaunch.configurers import AnsibleAppConfigurer from rest_framework.serializers import ValidationError log = get_task_logger('cloudlaunch') def get_required_val(data, name, message): val = data.get(name) if not val: raise ValidationError({"error": message}) return val class RancherKubernetesApp(BaseVMAppPlugin): """ Rancher Kubernetes Appliance. """ @staticmethod def validate_app_config(provider, name, cloud_config, app_config): rancher_config = get_required_val( app_config, "config_rancher_kube", "Rancher configuration data" " must be provided. config_rancher_kube entry not found in" " app_config.") #user_data = "#!/bin/bash\n" #user_data += get_required_val(
iterate_repeat_records, ) from corehq.privileges import DATA_FORWARDING, ZAPIER_INTEGRATION from corehq.util.metrics import make_buckets_from_timedeltas from corehq.util.soft_assert import soft_assert _check_repeaters_buckets = make_buckets_from_timedeltas( timedelta(seconds=10), timedelta(minutes=1), timedelta(minutes=5), timedelta(hours=1), timedelta(hours=5), timedelta(hours=10), ) _soft_assert = soft_assert(to='@'.join(('nhooper', 'dimagi.com'))) logging = get_task_logger(__name__) @periodic_task( run_every=crontab(day_of_month=27), queue=settings.CELERY_PERIODIC_QUEUE, ) def clean_logs(): """ Drop MOTECH logs older than 90 days. Runs on the 27th of every month. """ ninety_days_ago = datetime.now() - timedelta(days=90) RequestLog.objects.filter(timestamp__lt=ninety_days_ago).delete()
Transcript, clean_video_id, get_transcript_from_contentstore, TranscriptsGenerationException) from xmodule.modulestore import ModuleStoreEnum from xmodule.exceptions import NotFoundError from edxval.api import ( ValCannotCreateError, create_video_transcript, is_video_available, is_transcript_available, create_or_update_video_transcript, create_external_video, ) User = get_user_model() LOGGER = get_task_logger(__name__) FILE_READ_CHUNK = 1024 # bytes FULL_COURSE_REINDEX_THRESHOLD = 1 DEFAULT_ALL_COURSES = False DEFAULT_FORCE_UPDATE = False DEFAULT_COMMIT = False MIGRATION_LOGS_PREFIX = 'Transcript Migration' RETRY_DELAY_SECONDS = 30 COURSE_LEVEL_TIMEOUT_SECONDS = 1200 VIDEO_LEVEL_TIMEOUT_SECONDS = 300 @chord_task(bind=True) def task_status_callback( self,
def log(self): return get_task_logger("%s.%s" % (__name__, self.alert_name))
import mioji.common.pool import mioji.common.pages_store import pymongo import datetime import mioji.common import pymongo.errors from proj.my_lib.logger import func_time_logger from proj.list_config import cache_config, list_cache_path, cache_type, none_cache_config from proj.my_lib.Common.BaseSDK import BaseSDK from proj.my_lib.ServiceStandardError import ServiceStandardError from proj import config from mongo_pool import mongo_data_client from proj.my_lib.Common.Browser import proxy_pool mioji.common.pool.pool.set_size(2024) logger = get_task_logger('hotel_list') mioji.common.logger.logger = logger mioji.common.pages_store.cache_dir = list_cache_path mioji.common.pages_store.STORE_TYPE = cache_type # client = pymongo.MongoClient(host='10.10.213.148', maxPoolSize=20) # collections = client['data_result']['HotelList'] # pymongo client client = pymongo.MongoClient('mongodb://*****:*****@10.19.2.103:27017/') collections = client['data_result']['hotel_list'] filter_collections = client['data_result']['hotel_filter'] # 初始化工作 (程序启动时执行一次即可) insert_db = None # get_proxy = simple_get_socks_proxy get_proxy = proxy_pool.get_proxy
Vim�UnDo���p~&�vƽk�& ≅=���č���Dz�� class Gateway: ^j��_����� V ^j���& def __init__(self, response_json): try:) self.response = response_json, self.data = response_json.json() except Exception: self.data = {}5�_�����V^j��� def to_dict(self): return self.data5�_�����V^j���,from celery.utils.log import get_task_logger"logger = get_task_logger(__name__)5�_�����V^j���5�_�����V^j����5�_�����V^j���5�_�����V^j���=from coinmine.coin.apis.base_connection import BaseConnection5�_� ����V^j���3from coinmine.coin.apis.base_ import BaseConnection5�_� 0����V^j���:from coinmine.coin.apis.base_gateway import BaseConnection5�_� 0����V^j���0from coinmine.coin.apis.base_gateway import Base5�_� 6����V^j���7from coinmine.coin.apis.base_gateway import BaseGatewayclass Gateway: def status(self):- return self.data.get("status", False) def error(self):+ return self.data.get("error", None)5�_� ����V^j��� class Gateway:5��
from datetime import datetime, timedelta import requests from celery import group from celery.utils.log import get_task_logger from celery.exceptions import Ignore from tasks import settings from tasks.application import app from tasks.utils.session import session from tasks.utils.database import databases from tasks.errors.parse_error import ParseError, ParserNotFound from tasks.parsers import Parser logger = get_task_logger('tasks') # pylint: disable=C0103 mapping = { 'hotel': 'hotel_orders', 'flight': 'flight_orders', 'restaurant': 'restaurant_orders' } ai_base = 'https://ai.weego.me/api/v3/ai/user_profiles/' @app.task def refresh_access_token(email: str): # Type: Bool """Refresh access token by refresh token """ # scripture = databases('scripture') # user = scripture.g_users.find_one({'email': email})
from __future__ import absolute_import import urllib2 import json from celery import shared_task from celery.utils.log import get_task_logger from django.conf import settings from lrs.models import Activity, Statement from lrs.util import StatementValidator as SV celery_logger = get_task_logger('celery-task') @shared_task def check_activity_metadata(stmts): activity_ids = list( Activity.objects.filter( object_of_statement__statement_id__in=stmts).values_list( 'activity_id', flat=True).distinct()) [get_activity_metadata(a_id) for a_id in activity_ids] @shared_task def void_statements(stmts): try: Statement.objects.filter(statement_id__in=stmts).update(voided=True) except Exception, e: celery_logger.exception("Voiding Statement Error: " + e.message)
def get_logger(self, **kwargs): return get_task_logger(self.name)
import os import sys from celery.utils.log import get_task_logger from downloader_app.celeryapp import app from downloader_app.tiff_downloader import download_tiffs as td BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) WORK_DIR = os.path.join(BASE_DIR, 'downloader_app') sys.path.insert(0, WORK_DIR) logger = get_task_logger('downloader_app') @app.task def download_source(source, dates, point1, point2, opt=False): """ Download satelite tiff files and save it to the directory 'downloadedFiles'. """ try: logger.info('Fetch {} {}'.format(source, dates)) td(source, dates, point1, point2, opt) except Exception as e: logger.error( '[EE] fetching from {} at {} error: {}'.format(source, dates, e) ) return
from proj.my_lib.Common.Browser import proxy_pool from celery.utils.log import get_task_logger import mioji.common.logger import mioji.common.pool import mioji.common.pages_store import urlparse from lxml import html import re import time import pymongo from proj.my_lib.Common.Task import Task as Task_to import requests import traceback mioji.common.pool.pool.set_size(1024) logger = get_task_logger('daodaoHotel') mioji.common.logger.logger = logger mioji.common.pages_store.cache_dir = list_cache_path mioji.common.pages_store.STORE_TYPE = cache_type # 初始化工作 (程序启动时执行一次即可) insert_db = None # get_proxy = simple_get_socks_proxy get_proxy = proxy_pool.get_proxy debug = False spider_factory.config_spider(insert_db, get_proxy, debug, need_flip_limit=False) clients = pymongo.MongoClient(host='10.10.231.105') SourceIDS = clients['ImagesMD5']['SourceId'] def hotel_url_to_database(source, keyword, need_cache=False): task = Task()