def report_to_bigquery():
"""Flush all pending events of a certain type to BigQuery."""
# Schedule multiple flush jobs per minute for some events.
if request.method == 'GET':
tasks = []
for delay in xrange(0, 60, 5):
tasks.append(taskqueue.Task(method='POST', url=request.path,
countdown=delay,
params={'event_name': 'content_vote_v1'}))
tasks.append(taskqueue.Task(method='POST', url=request.path))
taskqueue.Queue(config.BIGQUERY_CRON_QUEUE_NAME).add(tasks)
return ''
# Retrieve pending events from pull queue.
try:
q = taskqueue.Queue(config.BIGQUERY_QUEUE_NAME)
tasks = q.lease_tasks_by_tag(config.BIGQUERY_LEASE_TIME.total_seconds(),
config.BIGQUERY_LEASE_AMOUNT,
tag=flask_extras.get_parameter('event_name'))
logging.debug('Leased %d event(s) from %s', len(tasks), config.BIGQUERY_QUEUE_NAME)
except taskqueue.TransientError:
logging.warning('Could not lease events due to transient error')
return '', 503
if not tasks:
return ''
# Insert the events into BigQuery.
table_id = tasks[0].tag
rows = [json.loads(t.payload) for t in tasks]
bigquery_client.insert_rows(table_id, rows)
# Delete the tasks now that we're done with them.
q.delete_tasks(tasks)
return ''
def get(self):
# Purge entire queue
q = taskqueue.Queue('updatequeue')
q.purge()
q = taskqueue.Queue('default')
q.purge()
self.redirect(webapp2.uri_for('home'))
def __notify_feature_subscribers_of_changes(self, is_update):
"""Async notifies subscribers of new features and property changes to features by
posting to a task queue."""
# Diff values to see what properties have changed.
changed_props = []
for prop_name, prop in self.properties().iteritems():
new_val = getattr(self, prop_name, None)
old_val = getattr(self, '_old_' + prop_name, None)
if new_val != old_val:
changed_props.append({
'prop_name': prop_name, 'old_val': old_val, 'new_val': new_val})
payload = json.dumps({
'changes': changed_props,
'is_update': is_update,
'feature': self.format_for_template(version=2)
})
# Create task to email subscribers.
queue = taskqueue.Queue()#name='emailer')
task = taskqueue.Task(method="POST", url='/tasks/email-subscribers',
target='notifier', payload=payload)
queue.add(task)
# Create task to send push notifications
queue = taskqueue.Queue()
task = taskqueue.Task(method="POST", url='/tasks/send_notifications',
target='notifier', payload=payload)
queue.add(task)
def prep(search_params=None, **kw):
if search_params is None:
return
if 'filters' not in search_params:
return
filters = search_params['filters']
if len(filters) != 1:
return
filter = filters[0]
if 'name' not in filter:
return
if filter['name'] != 'scholar_id':
return
if filter['op'] != '==':
return
scholar_id = filter['val']
publication = Publication.query.filter_by(scholar_id=scholar_id).first()
if (publication is None or publication.retrieved_at is None):
queue = taskqueue.Queue('publication-fetchers')
task = taskqueue.Task(url='/publication/crawl',
params={'scholar_id': scholar_id})
queue.add(task)
raise ProcessingException(description='Try later.', code=202)
elif (datetime.now() - publication.retrieved_at).days > 365:
queue = taskqueue.Queue('publication-fetchers')
task = taskqueue.Task(url='/publication/crawl',
params={'scholar_id': scholar_id})
queue.add(task)
def testAddingTasks(self):
"""Tests for adding tasks."""
taskqueue.add(url='/run')
taskqueue.Queue('test').add(taskqueue.Task(url='/foo'))
self.assertRaises(taskqueue.UnknownQueueError,
taskqueue.Queue('unknown').add,
taskqueue.Task(url='/foo'))
def post(self):
"""In this task, we should watch an episode entity
and check the task queue for the new completed
episode. then, create a new task to send the email. """
# TODO 1: in the first run, we shoul defer it for later, because if
# there were no task pending, it will be instantly trigguered :(
keyEpisode = self.request.get('keyEpisode')
submitter = self.request.get('submitter')
# get the episode from the database
# check if this episode has videos. (Trick for the earliest version)
# TODO: check if all this episode videos have been added or failed
# by now, just check if video queue is empty
queue = taskqueue.Queue() # get the queue of newVideos
queueStats = queue.fetch_statistics() # returns QueueStatistics
if queueStats.tasks == 0:
logging.debug("There are no pending video tasks")
# so send an email
sender = "Capitulizer Mighty Bot <*****@*****.**>"
to = submitter # send them to the submitter TODO
cc = "" # by now, empty string so sending doesnt fail
#bcc = "*****@*****.**" # and send a copy to the admin
subject = "New Episode Available to Watch"
# TODO 1: create the notification body
body = """<html><h1>Notification Email Body</h1>
http://capitulizer.appspot.com/</html>"""
# New version, creating a new task
logging.debug("Creating task sendEmail")
queue = taskqueue.Queue('sendEmail')
task = taskqueue.Task(url='/tasks/sendEmail',
params={
'sender': sender,
'subject': subject,
'body': body,
'cc': cc,
'bcc': bcc,
'to': to
})
queue.add(task)
logging.debug("Creating task newPost")
queue = taskqueue.Queue('newPost')
task = taskqueue.Task(url='/tasks/newPost',
params={'keyEpisode': keyEpisode})
queue.add(task)
else:
logging.debug("There are still some video tasks to complete")
# so retry
raise Exception("Still new videos waiting. Maybe next time...")
def get(self):
self.pull = taskqueue.Queue('recordio-queue')
tag = self.request.get("taskqueue")
max_tasks_to_lease = MAX_RPC_SIZE / MAX_TASKQUEUE_BATCH_SIZE
if tag:
batch = []
batch_size = 0
success = True
while True:
tasks = self.pull.lease_tasks_by_tag(LEASE_TIME_PER_BATCH,
max_tasks_to_lease,
tag=tag)
for task in tasks:
if task.was_deleted:
# Should never happend
continue
next_key_values = marshal.loads(task.payload)
next_size = sum(
[recordio_chunks.size(x) for x in next_key_values])
if next_size + batch_size >= MAX_WRITE_BATCH_SIZE:
success = success and self.commit_batch(tag, batch)
batch = [(task, next_key_values)]
batch_size = next_size
else:
batch_size += next_size
batch.append((task, next_key_values))
if len(tasks) != max_tasks_to_lease:
break
success = success and self.commit_batch(tag, batch)
if not success:
raise Exception("RecordIO not completed")
else:
pending_tasks = self.pull.lease_tasks(0, max_tasks_to_lease)
seen = set([])
for task in pending_tasks:
tag = task.tag
if tag in seen:
continue
seen.add(tag)
try:
taskqueue.Queue('recordio-writer').add(
RecordIOWriter.create_task_(tag, in_past=True))
self.response.out.write("Scheduled write for: %s<br>" %
tag)
except (taskqueue.DuplicateTaskNameError,
taskqueue.TombstonedTaskError,
taskqueue.TaskAlreadyExistsError):
self.response.out.write(
"Already pending write for: %s<br>" % tag)
if len(pending_tasks) == max_tasks_to_lease:
self.response.out.write(
"<script type=text/javascript>window.setTimeout(function() {"
"document.location.reload();"
"}, 5000);</script>")
def _insert_tasks(tasks,
queue,
transactional=False,
retry_transient_errors=True,
retry_delay=RETRY_SLEEP_SECS):
"""Insert a batch of tasks into the specified queue. If an error occurs
during insertion, split the batch and retry until they are successfully
inserted. Return the number of successfully inserted tasks.
"""
from google.appengine.api import taskqueue
if not tasks:
return 0
try:
taskqueue.Queue(name=queue).add(tasks, transactional=transactional)
return len(tasks)
except (taskqueue.BadTaskStateError, taskqueue.TaskAlreadyExistsError,
taskqueue.TombstonedTaskError):
if len(tasks) <= 1:
# Task has already been inserted, no reason to report an error here.
return 0
# If a list of more than one Tasks is given, a raised exception does
# not guarantee that no tasks were added to the queue (unless
# transactional is set to True). To determine which tasks were
# successfully added when an exception is raised, check the
# Task.was_enqueued property.
reinsert = _tasks_to_reinsert(tasks, transactional)
count = len(reinsert)
inserted = len(tasks) - count
inserted += _insert_tasks(reinsert[:count / 2], queue, transactional,
retry_transient_errors, retry_delay)
inserted += _insert_tasks(reinsert[count / 2:], queue, transactional,
retry_transient_errors, retry_delay)
return inserted
except taskqueue.TransientError:
# Always re-raise for transactional insert, or if specified by
# options.
if transactional or not retry_transient_errors:
raise
reinsert = _tasks_to_reinsert(tasks, transactional)
# Retry with a delay, and then let any errors re-raise.
time.sleep(retry_delay)
taskqueue.Queue(name=queue).add(reinsert, transactional=transactional)
return len(tasks)
def consume_pull(self):
if "X-AppEngine-Cron" in self.request.headers:
try:
q = taskqueue.Queue(self.queue_name)
stats = q.fetch_statistics()
task_in_queue = stats.tasks
logging.error(str(task_in_queue))
count = task_in_queue / 20000.0
while count > 0:
url = '/openxc_stats/consume_pull'
taskqueue.add(queue_name='general-stats', url=url)
count -= 1
except:
logging.error(traceback.format_exc())
return
lease_seconds = 10
max_tasks = 1000
num_merge = 100
start_time = time.time()
q = taskqueue.Queue(self.queue_name)
while True:
if time.time() - start_time > 50:
break
try:
tasks = q.lease_tasks(lease_seconds, max_tasks)
if len(tasks) == 0:
return
except:
logging.error(traceback.format_exc())
return
task_sub_lists = [
tasks[i:i + int(num_merge)]
for i in range(0, len(tasks), int(num_merge))
]
for sub_list in task_sub_lists:
entities = []
for task in sub_list:
entities.append(
OpenXCStats.get_datastore_entity(
json.loads(task.payload)))
try:
db.put(entities)
q.delete_tasks(sub_list)
except:
logging.error(traceback.format_exc())
def post(self):
if not self.request.body:
self.response.write("Bad request.")
self.response.status_int = 400
return
dna = json.loads(self.request.body)
if not self.valid_dna(dna.get('dna')):
self.response.write("Bad request.")
self.response.status_int = 400
return
is_mutant = Human.is_mutant(dna.get('dna'))
queue = taskqueue.Queue(name='default')
task = taskqueue.Task(url='/insert_dna/',
target='worker',
params={
'dna': dna,
'is_mutant': is_mutant
})
queue.add(task)
if is_mutant:
self.response.write('It is mutant.')
else:
self.response.write("Is human.")
self.response.status_int = 403
def cancel_tasks(self):
task_namespace = self._get_task_namespace()
enqueued_tasks = TaskEnqueued.where(task_namespace=task_namespace)
if enqueued_tasks:
tasks = [taskqueue.Task(name=t.task_name) for t in enqueued_tasks]
taskqueue.Queue().delete_tasks(tasks)
TaskEnqueued.where(task_namespace=task_namespace).delete()
def ProcessHistogramSet(histogram_dicts):
if not isinstance(histogram_dicts, list):
raise BadRequestError('HistogramSet JSON much be a list of dicts')
histograms = histogram_module.HistogramSet()
histograms.ImportDicts(histogram_dicts)
histograms.ResolveRelatedHistograms()
InlineDenseSharedDiagnostics(histograms)
revision = ComputeRevision(histograms)
task_list = []
suite_path = ComputeSuitePath(histograms)
for diagnostic in histograms.shared_diagnostics:
# We'll skip the histogram-level sparse diagnostics because we need to
# handle those with the histograms, below, so that we can properly assign
# test paths.
if type(diagnostic) in SUITE_LEVEL_SPARSE_DIAGNOSTIC_TYPES:
task_list.append(_MakeTask(diagnostic, suite_path, revision))
for histogram in histograms:
guid = histogram.guid
objects = FindHistogramLevelSparseDiagnostics(guid, histograms)
test_path = ComputeTestPath(guid, histograms)
# We need to queue the histogram in addition to its dense shared
# diagnostics.
objects.append(histogram)
# TODO(eakuefner): Batch these better than one per task.
for obj in objects:
task_list.append(_MakeTask(obj, test_path, revision))
queue = taskqueue.Queue(TASK_QUEUE_NAME)
queue.add(task_list)
def manual_process_for_venue(venue_id):
raise DeprecationWarning(
'this code is now depricated as it relies on the depricated InstagramDataProcessor'
)
logging.debug("SPICE: socialnetworks/instagram/process_for_venue started")
if instagram_settings.instagram_auto_process_callback == False:
instagram_queue = taskqueue.Queue('instagramcallback')
queue_stats = instagram_queue.fetch_statistics()
# Only add if there is no task in the queue
if queue_stats.tasks < 1:
task = taskqueue.Task(
url=
'/socialengine/api/instagram/callback_task?key=%s&venue_id=%s'
% (spice_settings.static_api_key, venue_id),
method='GET')
instagram_queue.add(task)
logging.debug(
"SPICE: socialnetworks/instagram/process_for_venue added task for venue_id %s"
% venue_id)
logging.debug("SPICE: socialnetworks/instagram/process_for_venue ended")
return None
def get(self):
"""Indefinitely fetch tasks and update the datastore."""
queue = taskqueue.Queue('pullq')
while True:
try:
tasks = queue.lease_tasks_by_tag(3600, 1000, deadline=60)
except (taskqueue.TransientError,
apiproxy_errors.DeadlineExceededError) as e:
logging.exception(e)
time.sleep(1)
continue
if tasks:
key = tasks[0].tag
@ndb.transactional
def update_counter():
counter = Counter.get_or_insert(key, count=0)
counter.count += len(tasks)
counter.put()
try:
update_counter()
except Exception as e:
logging.exception(e)
else:
queue.delete_tasks(tasks)
time.sleep(1)
def cron_run_import(): # pragma: no cover
"""Schedules a push task for each config set imported from Gitiles."""
conf = admin.GlobalConfig.fetch()
# Collect the list of config sets to import.
config_sets = []
if (conf and conf.services_config_storage_type == GITILES_STORAGE_TYPE
and conf.services_config_location):
loc = _resolved_location(conf.services_config_location)
config_sets += _service_config_sets(loc)
config_sets += _project_and_ref_config_sets()
# For each config set, schedule a push task.
# This assumes that tasks are processed faster than we add them.
tasks = [
taskqueue.Task(url='/internal/task/luci-config/gitiles_import/%s' % cs)
for cs in config_sets
]
# Task Queues try to preserve FIFO semantics. But if something is partially
# failing (e.g. LUCI Config hitting gitiles quota midway through update), we'd
# want to make a slow progress across all config sets. Shuffle tasks, so we
# don't give accidental priority to lexicographically first ones.
random.shuffle(tasks)
q = taskqueue.Queue('gitiles-import')
pending = tasks
while pending:
batch = pending[:100]
pending = pending[len(batch):]
q.add(batch)
logging.info('scheduled %d tasks', len(tasks))
def _insert_tasks(tasks, queue, transactional=False):
"""Insert a batch of tasks into the specified queue. If an error occurs
during insertion, split the batch and retry until they are successfully
inserted. Return the number of successfully inserted tasks.
"""
from google.appengine.api import taskqueue
if not tasks:
return 0
try:
taskqueue.Queue(name=queue).add(tasks, transactional=transactional)
return len(tasks)
except (taskqueue.BadTaskStateError,
taskqueue.TaskAlreadyExistsError,
taskqueue.TombstonedTaskError,
taskqueue.TransientError):
count = len(tasks)
if count <= 1:
return 0
inserted = _insert_tasks(tasks[:count / 2], queue, transactional)
inserted += _insert_tasks(tasks[count / 2:], queue, transactional)
return inserted
def post(self):
_msg = self.request.get('message', '0')
if _msg is not None:
sentOn = None
_type = self.request.get('type', '')
post_args = self.request.arguments()
if _type in ['feedback', 'error-feedback']:
if 'reportsentutc' in post_args:
sentOn = self.parseDateTime(
self.request.get('reportsentutc', ''))
_groupId = long(self.request.get('groupid', ''))
if _groupId and sentOn:
if _msg != 'Automatically sent':
fb = Feedback(groupId=_groupId,
sendTime=sentOn,
timezone=self.request.get(
'reportsenttz', ''),
type=_type,
message=_msg)
fb.put()
q = taskqueue.Queue('feedback-export-queue')
q.add((taskqueue.Task(payload=fb.getExportLine(),
method='PULL')))
Cnt.incr("Feedback_counter")
else:
logging.warning(
'Automatically sent feedback message, discarding.')
self.response.out.write("OK")
else:
self.error(400)
else:
self.error(400)
else:
self.error(400)
def handle_get(self):
queue = taskqueue.Queue('update')
if queue.fetch_statistics().tasks > 0:
self.response.write('update already in progress')
return
query = Library.query()
cursor = None
more = True
task_count = 0
while more:
keys, cursor, more = query.fetch_page(50,
keys_only=True,
start_cursor=cursor)
for key in keys:
task_count = task_count + 1
task_url = util.update_library_task(key.id())
util.new_task(task_url, target='manage', queue_name='update')
logging.info('triggered %d library updates', task_count)
query = Author.query()
cursor = None
more = True
task_count = 0
while more:
keys, cursor, more = query.fetch_page(50,
keys_only=True,
start_cursor=cursor)
for key in keys:
task_count = task_count + 1
task_url = util.update_author_task(key.id())
util.new_task(task_url, target='manage', queue_name='update')
logging.info('triggered %d author updates', task_count)
def ProcessHistogramSet(histogram_dicts):
if not isinstance(histogram_dicts, list):
raise api_request_handler.BadRequestError(
'HistogramSet JSON much be a list of dicts')
histograms = histogram_set.HistogramSet()
histograms.ImportDicts(histogram_dicts)
histograms.ResolveRelatedHistograms()
InlineDenseSharedDiagnostics(histograms)
revision = ComputeRevision(histograms)
task_list = []
suite_key = GetSuiteKey(histograms)
suite_level_sparse_diagnostic_entities = []
diagnostic_names_added = {}
# We'll skip the histogram-level sparse diagnostics because we need to
# handle those with the histograms, below, so that we can properly assign
# test paths.
for hist in histograms:
for name, diag in hist.diagnostics.iteritems():
if name in SUITE_LEVEL_SPARSE_DIAGNOSTIC_NAMES:
if diagnostic_names_added.get(name) is None:
diagnostic_names_added[name] = diag.guid
if diagnostic_names_added.get(name) != diag.guid:
raise ValueError(
name +
' diagnostics must be the same for all histograms')
if name in SUITE_LEVEL_SPARSE_DIAGNOSTIC_NAMES:
suite_level_sparse_diagnostic_entities.append(
histogram.SparseDiagnostic(id=diag.guid,
data=diag.AsDict(),
test=suite_key,
start_revision=revision,
end_revision=sys.maxint,
name=name))
# TODO(eakuefner): Refactor master/bot computation to happen above this line
# so that we can replace with a DiagnosticRef rather than a full diagnostic.
new_guids_to_old_diagnostics = DeduplicateAndPut(
suite_level_sparse_diagnostic_entities, suite_key, revision)
for new_guid, old_diagnostic in new_guids_to_old_diagnostics.iteritems():
histograms.ReplaceSharedDiagnostic(
new_guid, diagnostic.Diagnostic.FromDict(old_diagnostic))
for hist in histograms:
guid = hist.guid
diagnostics = FindHistogramLevelSparseDiagnostics(guid, histograms)
# TODO(eakuefner): Don't compute full diagnostics, because we need anyway to
# call GetOrCreate here and in the queue.
test_path = ComputeTestPath(guid, histograms)
# TODO(eakuefner): Batch these better than one per task.
task_list.append(_MakeTask(hist, test_path, revision, diagnostics))
queue = taskqueue.Queue(TASK_QUEUE_NAME)
queue.add(task_list)
def processOfRequest(self):
# # cronからのコールかどうかを判定
# if self.getServerVariables('X-AppEngine-Cron') != 'true':
# logging.error('invalid access it does not accessed by any cron.')
# return
# target = backends.get_backend()
# logging.info('backends_target=' + (str(target) if backends is not None else ''))
# テナントごとにループ ※namespace_managerから取得できれば一番いいがとりあえずDomainEntryをループ
tenant_entrys = sateraito_func.get_all_tenant_entry()
for tenant_entry in tenant_entrys:
###############################################
# ドメイン確定＆ネームスペースセット（queueはキュー追加時点のネームスペースにて実行されるとのこともありここでセット）
tenant = tenant_entry.tenant
# 無効テナントは処理しない
if sateraito_func.isTenantDisabled(tenant):
pass
else:
# token作成
token = UcfUtil.guid()
# Save Number of GoogleApps domain user
params = {'requestor': '', 'type': 'start'}
# taskに追加 まるごと
import_q = taskqueue.Queue('tenant-set-queue')
import_t = taskqueue.Task(
url='/a/' + tenant + '/openid/' + token +
'/regist_tenant_entry',
params=params,
#target='b1process', # 365版はGAE側でタスク実行しないのでFrontEndsに変更 2015.03.09
target='',
countdown='5')
#logging.info('run task')
import_q.add(import_t)
def queueAggregation(error, instance, backtraceText):
"""Enqueues a task to aggregate the given instance in to the given error."""
payload = {'error': str(error.key()), 'instance': str(instance.key()), 'backtrace': backtraceText}
taskqueue.Queue('aggregation').add([
taskqueue.Task(payload = json.dumps(payload), method='PULL')
])
queueAggregationWorker()
def get_queue(queue_name):
global queue_cache
try:
queue = queue_cache[queue_name]
except KeyError:
queue = queue_cache[queue_name] = taskqueue.Queue(queue_name)
return queue
def tick(request):
queue = taskqueue.Queue('challenges-sendemails')
task = taskqueue.Task(url='/challenges/send_challenge_emails',
method='GET')
queue.add(task)
return HttpResponse(json.dumps({'status': 'ok'}),
content_type='application/json')
def get(self):
if self.request.get("method") == "task" and self.request.get("summonername"):
if self.request.get("flag"):
sname = self.request.get("summonername").encode()
taskqueue.add(
url="/analyzer",
target="analyzerservice",
queue_name="summoner-analyzer",
params={"summonername": sname,
"flag":"true"}
)
else:
sname = self.request.get("summonername").encode()
taskqueue.add(
url = "/analyzer",
target = "analyzerservice",
queue_name = "summoner-analyzer",
params={"summonername": sname}
)
self.response.write("%s has been added to analyzer_service"%(self.request.get("summonername")))
else:
statsList = taskqueue.QueueStatistics.fetch(taskqueue.Queue("summoner-analyzer"))
self.response.write(html)
self.response.write("<p>"+str(statsList)+"</p><hr>")
apistats = memcache.get(key="riot")
if apistats:
self.response.write('<br><p>API calls in the last minute: <div id="int">%d calls</div></p>'%(apistats))
else:
self.response.write('<br><p>API calls in the last minute: <div id="int">No data available(check memcache)</div></p>')
self.response.write(endhtml)
def create_videos_tasks(self, linksInter=[], keyEpisode=""):
try:
# create newVideo tasks to add videos to the episode.
limit = 30 # TODO eliminate this limit. is horrible, sucker
#if we want to set a limit, we should do it better
# with the interlinks, get some data
for linkInter in linksInter:
# TODO: rename linkInter to interLink
limit -= 1
# build a link if its not complete
linkInter = extract.buildLink(linkInter)
# and dont get past the limit
if limit > 0:
# Call a newVideo task for each interlink
queue = taskqueue.Queue('newVideo')
task = taskqueue.Task(url='/tasks/newVideo',
params={
'keyEpisode': keyEpisode,
'interLink': linkInter
})
queue.add(task)
else:
logging.error("We still have a limit set")
break
except:
raise # TODO
def AuthorizedPost(self):
datastore_hooks.SetPrivilegedRequest()
with timing.WallTimeLogger('decompress'):
try:
data_str = self.request.body
zlib.decompress(data_str)
logging.info('Recieved compressed data.')
except zlib.error:
data_str = self.request.get('data')
data_str = zlib.compress(data_str)
logging.info('Recieved uncompressed data.')
if not data_str:
raise api_request_handler.BadRequestError(
'Missing "data" parameter')
filename = uuid.uuid4()
params = {'gcs_file_path': '/add-histograms-cache/%s' % filename}
gcs_file = cloudstorage.open(params['gcs_file_path'],
'w',
content_type='application/octet-stream',
retry_params=_RETRY_PARAMS)
gcs_file.write(data_str)
gcs_file.close()
retry_options = taskqueue.TaskRetryOptions(
task_retry_limit=_TASK_RETRY_LIMIT)
queue = taskqueue.Queue('default')
queue.add(
taskqueue.Task(url='/add_histograms/process',
payload=json.dumps(params),
retry_options=retry_options))
def cron_run_import(): # pragma: no cover
"""Schedules a push task for each config set imported from Gitiles."""
conf = admin.GlobalConfig.fetch()
# Collect the list of config sets to import.
config_sets = []
if (conf and conf.services_config_storage_type == GITILES_STORAGE_TYPE and
conf.services_config_location):
loc = gitiles.Location.parse_resolve(conf.services_config_location)
config_sets += _service_config_sets(loc)
config_sets += _project_and_ref_config_sets()
# For each config set, schedule a push task.
# This assumes that tasks are processed faster than we add them.
tasks = [
taskqueue.Task(url='/internal/task/luci-config/gitiles_import/%s' % cs)
for cs in config_sets
]
q = taskqueue.Queue('gitiles-import')
pending = tasks
while pending:
batch = pending[:100]
pending = pending[len(batch):]
q.add(batch)
logging.info('scheduled %d tasks', len(tasks))
def delete(self):
queue_name = self.request.get('queueName')
task_id = self.request.get('taskId')
queue = taskqueue.Queue(queue_name)
task = taskqueue.Task(name=task_id)
queue.delete_tasks(task)
def EnqueueTasks(tasks, task_tag):
"""Enqueues a list of tasks in the Google Cloud task queue, for consumption by
Google Compute Engine.
"""
q = taskqueue.Queue('clovis-queue')
# Add tasks to the queue by groups.
# TODO(droger): This supports thousands of tasks, but maybe not millions.
# Defer the enqueuing if it times out.
group_size = 100
callbacks = []
try:
for i in range(0, len(tasks), group_size):
group = tasks[i:i + group_size]
taskqueue_tasks = [
taskqueue.Task(payload=task.ToJsonString(),
method='PULL',
tag=task_tag) for task in group
]
rpc = taskqueue.create_rpc()
q.add_async(task=taskqueue_tasks, rpc=rpc)
callbacks.append(rpc)
for callback in callbacks:
callback.get_result()
except Exception as e:
clovis_logger.error('Exception:' + type(e).__name__ + ' ' +
str(e.args))
return False
clovis_logger.info('Pushed %i tasks with tag: %s.' %
(len(tasks), task_tag))
return True
def get(self):
q = taskqueue.Queue('views')
tasks = q.lease_tasks(360, 100)
if not tasks:
logging.info('No views to update')
return
stats = {}
for task in tasks:
payload = json.loads(task.payload)
url_id = payload['id']
if url_id not in stats:
stats.update({url_id: {'one': 0, 'two': 0}})
if payload['index'] == 1:
stats[payload['id']]['one'] += 1
elif payload['index'] == 2:
stats[payload['id']]['two'] += 1
for u_id in stats.iterkeys():
url = RandomURL.get_by_id(u_id)
url.views_for_one = stats[u_id]['one']
url.views_for_two = stats[u_id]['two']
url.all_views = stats[u_id]['one'] + stats[u_id]['two']
url.put()
q.delete_tasks(tasks)
