def publish(subject, text): ''' Publish cumulative post to Livejournal. ''' r = ResQ() for target in settings.PUBLISHING_LJ_TARGETS.iterkeys(): r.enqueue(LivejournalPost, target, subject, text)
def publish(text, url=None): ''' Publish status to social networks. ''' r = ResQ() r.enqueue(TwitterStatus, text, url) r.enqueue(VkontakteStatus, text, url)
def perform(eventid): print (eventid) db = sqlitedb() eventrec = db.getactiveevent() for event in eventrec: print event['eventid'] r = ResQ(server="%s:%s" % (resqserver, resqport)) r.enqueue(callQ, event['eventid'])
def trigger(self, timer): intent_json = self.db.hget('timer:%s' % timer, 'intent') if not intent_json: logging.warning("Triggering timer %s, but found no intent" % timer) return from async .send_intent import SendIntent r = ResQ() r.enqueue(SendIntent, intent_json)
def do_start_build(job_name): input = request.json job = Job.load(job_name, input.get('job_ref')) build = Build.create(job, parameters=input.get('parameters', {}), description=input.get('description', '')) session_id = '%s-0' % build.uuid set_session_queued(g.db, session_id) r = ResQ() r.enqueue(DispatchSession, session_id) return jsonify(**build.as_dict())
def dispatch(): input = request.json session_no = create_session(g.db, input['build_id'], parent=input['parent'], labels=input['labels'], run_info=input['run_info'], state=SESSION_STATE_TO_BACKEND) session_id = '%s-%s' % (input['build_id'], session_no) ri = input['run_info'] or {} args = ", ".join(ri.get('args', [])) title = "%s(%s)" % (ri.get('step_name', 'main'), args) item = RunAsync(session_no, title) add_slog(g.db, input['parent'], item) r = ResQ() r.enqueue(DispatchSession, session_id) return jsonify(session_id=session_id)
def check_in_available(agent_id): session_id = request.json['session_id'] build_id, num = session_id.split('-') with g.db.pipeline() as pipe: set_session_done(pipe, session_id, request.json['result'], request.json['output'], request.json['log_file']) if int(num) == 0: Build.set_done(build_id, request.json['result'], pipe=pipe) add_slog(pipe, session_id, SessionDone(request.json['result'])) pipe.hmset(jdb.KEY_AGENT % agent_id, dict(state=jdb.AGENT_STATE_AVAIL, seen=get_ts())) pipe.execute() r = ResQ() r.enqueue(AgentAvailable, agent_id) return jsonify()
def details(): r = ResQ() with closing(get_connection()) as connection: total = 0 with closing(connection.cursor()) as cursor: query = ''' SELECT COUNT(id) AS count FROM records WHERE details IS NULL ''' cursor.execute(query) total = cursor.fetchone()['count'] with closing(connection.cursor('cursor')) as cursor: query = ''' SELECT * FROM records WHERE details IS NULL ''' cursor.execute(query) for record in tqdm(cursor, total=total): r.enqueue(Record, record['id'])
def submit_feedback(request): if request.method != 'POST': raise Http404 if 'lang' in request.GET and request.GET['lang'] in ALLOWED_LANGS: translation.activate(request.GET['lang']) try: form = FeedbackForm(request.POST or None) if form.is_valid(): r = ResQ() r.enqueue(FeedbackTask, form.cleaned_data['name'], form.cleaned_data['email'], form.cleaned_data['text']) return HttpResponse(status=200) # form is invalid return HttpResponse( loader.render_to_string('feedback/feedback_form_partial.html', {'feedback_form': form}), status=409) except Exception: logger.exception('') raise
def add_repository_to_queue(user, repo_owner, repository_name, repo_object=None): res = ResQ() repo = None token = user.get_profile().extra_data['access_token'] if not repo_object: gh = Github(login=user.email, token=token) repo = gh.repos.get(repo_owner, repository_name) else: repo = repo_object queue_data = {'email': user.email, 'token': token} queue_data['repo'] = { 'name': repo.name, 'url': repo.url, 'git_url': repo.git_url, 'html_url': repo.html_url, 'language': repo.language } db_repo, created = Repository.objects.get_or_create( git_url=repo.git_url, defaults=queue_data['repo']) query_filter = {'user': user, 'repository': db_repo} query_filter.update({'defaults': { 'user': user, "repository": db_repo } }) as_contributor, created = Contributor.objects.get_or_create(**query_filter) # put a timestamp field in repository model to verify if there is need to # process again. so only put in queue if is not created and timestamp > x time res.enqueue(RepositoryWorker, queue_data)
def do_register(): agent_id = request.json['id'] info = { "ip": request.remote_addr, 'nick': request.json.get('nick', ''), "port": request.json["port"], "state": jdb.AGENT_STATE_AVAIL, "seen": get_ts(), "labels": ",".join(request.json["labels"]) } with g.db.pipeline() as pipe: pipe.hmset(jdb.KEY_AGENT % agent_id, info) pipe.sadd(jdb.KEY_ALL, agent_id) for label in request.json["labels"]: pipe.sadd(jdb.KEY_LABEL % label, agent_id) pipe.execute() r = ResQ() r.enqueue(AgentAvailable, agent_id) return jsonify()
def perform(text, url): ''' Delayed task. ''' if not settings.PUBLISHING_VKONTAKTE_USER: return if not url is None: text = '%s %s' % (text, url) method = 'wall.post' owner_id, token, extra_params = settings.PUBLISHING_VKONTAKTE_USER params = {'owner_id': owner_id, 'access_token': token, 'message': text} params.update(extra_params) resp = vk(method, params) post_id = resp.get("response", {}).get("post_id") if not post_id: raise ValueError('Post ID not provided.\nMethod %s\nParams %r'\ '\n Response %r' % (method, params, resp)) r = ResQ() for user, data in settings.PUBLISHING_VKONTAKTE_REPOST_USERS.iteritems(): r.enqueue(VkontakteLike, user, owner_id, post_id)
def queue_job(input_coordinate=None, \ hg18_option='off', \ transcript_type='NA', \ refseq_correspondence='no', \ sample_id='yes', \ analysis_type=None, \ analysis_programs=None): if analysis_type == None: print 'Choose an analysis type.' return input_basename = input_coordinate + '_' + hg18_option + '_' + transcript_type + '_' + refseq_correspondence + '_' + sample_id + '.txt' job_name = input_basename[:-4] + '_' + analysis_type + '_' + analysis_programs + '_20121231_010101' job_dir = os.path.join(diagnostic_tests_dir, job_name) if os.path.exists(job_dir) == False: os.mkdir(job_dir) input_filename = os.path.join(job_dir, input_basename) shutil.copy(os.path.join(diagnostic_tests_dir, input_basename), input_filename) job_id = job_name mutation_filename = input_filename mutation_filename_fix = mutation_filename + '.fix' mutation_filename_error = mutation_filename +'.error' email = '*****@*****.**' classifier = '_Other' upload_filename = input_basename user_upload_dir = job_dir chosen_db_str = analysis_programs.strip('_') chosen_dbs = chosen_db_str.split('_') tsv_report = 'on' gene_annot = 'on' hg18 = hg18_option functional_annot = 'on' f = open(input_filename) no_input_line = 0 for line in f: if len(line) > 0: if not line[0] in ['#', '>', '!']: no_input_line += 1 f.close() if no_input_line >= 5000: queue_name = queue_name_prefix + '_LARGE' else: queue_name = queue_name_prefix + '_SMALL' mupit_input = 'on' resubmit = 'no' wf = open(os.path.join(job_dir, 'job_info.txt'), 'w') wf.write(str(no_input_line)+'\n') wf.write(job_id+'\n') wf.write(chosen_db_str+'\n') wf.write('call_queuer_path\n') wf.write('python_path\n') wf.write('queuer_path\n') wf.write(email+'\n') wf.write(job_id+'\n') wf.write(classifier+'\n') wf.write(mutation_filename+'\n') wf.write(upload_filename+'\n') wf.write(user_upload_dir+'\n') wf.write(chosen_db_str+'\n') wf.write(tsv_report+'\n') wf.write(functional_annot+'\n') wf.write(hg18+'\n') wf.write(analysis_type+'\n') wf.write(str(no_input_line)+'\n') wf.write('error_output_path\n') wf.write(mupit_input+'\n') wf.close() r=ResQ() argstr = 'dummy' argstr += ' ' + 'yes' argstr += ' ' + user_upload_dir argstr += ' ' + resubmit argstr += ' ' + job_id argstr += ' -e ' + email argstr += ' -i ' + job_id argstr += ' -c ' + classifier argstr += ' -m ' + mutation_filename argstr += ' -u ' + upload_filename argstr += ' -D ' + user_upload_dir argstr += ' -d ' + chosen_db_str argstr += ' -t ' + tsv_report argstr += ' -f ' + functional_annot argstr += ' -r ' + hg18 argstr += ' -y ' + analysis_type argstr += ' -n ' + str(no_input_line) argstr += ' -M ' + mupit_input argstr += ' -R ' + resubmit argstr += ' -Q ' + queue_name argstr += ' -T ' + 'yes' # -T option for 'test' r.enqueue(queue_name, masteranalyzer.MasterAnalyzer, argstr) global job_ids job_ids.append(job_id)
class Task(object): queue ="test-queue" def __init__(self,payload): self.payload = payload @staticmethod def perform(json_dir): print "do something %s\n" % (json_dir) if __name__ == '__main__': r = ResQ() print("Enqueue message\n") r.enqueue(Task,"{key:id-1, file:'/tmp/toto.jpg'}") #print("get message from basic\n") #job = Job.reserve('test-queue', r) job = None if job: print("job payload %s\n" % (job._payload) ) else: print( "No jobs \n")
def publish(text, link, image): r = ResQ() r.enqueue(FacebookLink, text, link, image, True) r.enqueue(FacebookLink, text, link, image, False)