def main():
args = parse_args()
log = logging.getLogger("nuget-meta")
logging.basicConfig(level=logging.INFO)
host, port = args.beanstalkd.split(":")
client = BeanstalkClient(host, int(port), auto_decode=True)
try:
with open(args.output, "a") as fout:
for job in client.reserve_iter():
try:
execute_job(job.job_data, args.extractor, args.tmp, fout,
log)
except Exception:
log.exception(job)
try:
client.bury_job(job.job_id)
except Exception as e:
log.error("bury %s: %s: %s", job.job_data,
type(e).__name__, e)
continue
try:
client.delete_job(job.job_id)
except Exception as e:
log.error("delete %s: %s: %s", job.job_data,
type(e).__name__, e)
finally:
shutil.rmtree(os.path.join(args.tmp, str(os.getpid())),
ignore_errors=True)
def delete_buried(server, port, tube, jobCounter):
client = BeanstalkClient(server, port)
client.watch(tube)
client.use(tube)
while True:
try:
for job in client.peek_buried_iter():
client.delete_job(job.job_id)
jobCounter.increment_buried()
kicked = jobCounter.buried()
time.sleep(1)
if kicked == jobCounter.buried():
break
except BeanstalkError:
break
except KeyboardInterrupt:
break
def frombeanstalk(self):
try:
result = []
a = 0
result.append("\n--\t*Beanstalk*\t--")
while True:
tw = re.compile('sc_twitter_')
fb = re.compile('sc_fb_')
ig = re.compile('sc_ig_')
yt = re.compile('sc_youtube_')
if a == 1:
tube_list_fb = []
tube_list_ig = []
tube_list_yt = []
client = BeanstalkClient(self.config.get('ip_beanstalk','ip_ph'),self.config.get('ip_beanstalk','port_ph'))
list_tube = client.list_tubes()
for b in range(list_tube.__len__()):
if fb.findall(list_tube[b]): tube_list_fb.append(list_tube[b])
if ig.findall(list_tube[b]): tube_list_ig.append(list_tube[b])
if yt.findall(list_tube[b]): tube_list_yt.append(list_tube[b])
tube_list = tube_list_fb + tube_list_ig + tube_list_yt
for c in tube_list:
data = client.stats_tube(c)
if data['current-jobs-urgent'] >= 3000 or data['current-jobs-ready'] >= 3000:
res = "\n*{0}*\nworker : {1}\njobs-urgent : {2}\njobs-ready : {3}\njobs-buried : {4}\njobs-reserved: {5}" \
.format(data['name'], data['current-watching'], data['current-jobs-urgent'],
data['current-jobs-ready'], data['current-jobs-buried'], data['current-jobs-reserved'])
result.append(res)
else: continue
elif a == 0 :
tube_list = []
client = BeanstalkClient(self.config.get('ip_beanstalk','ip_bintaro'),self.config.get('ip_beanstalk','port_bintaro'))
list_tube = client.list_tubes()
for b in range(list_tube.__len__()):
if tw.findall(list_tube[b]): tube_list.append(list_tube[b])
for c in tube_list:
data = client.stats_tube(c)
if data['current-jobs-urgent'] >= 3000 or data['current-jobs-ready'] >= 3000:
res = "\n*{0}*\nworker : {1}\njobs-urgent : {2}\njobs-ready : {3}\njobs-buried : {4}\njobs-reserved: {5}" \
.format(data['name'], data['current-watching'], data['current-jobs-urgent'],
data['current-jobs-ready'], data['current-jobs-buried'], data['current-jobs-reserved'])
result.append(res)
else: continue
else: break
a+=1
if result.__len__() <= 1: result.append("\nBeanstalk Safe.")
print ("\n".join(result))
self.final['beanstalk'] = result
# return result
except ValueError as e: self.final['beanstalk'] = e
def delete_ready(server, port, tube, jobCounter):
client = BeanstalkClient(server, port)
client.watch(tube)
client.use(tube)
while True:
try:
for job in client.reserve_iter():
client.delete_job(job.job_id)
jobCounter.increment_ready()
processed = jobCounter.abs()
time.sleep(1)
if processed == jobCounter.abs():
break
except BeanstalkError:
break
except KeyboardInterrupt:
break
else:
app.secret_key = urandom(12)
client = MongoClient("mongodb://localhost:27017")
db = client["cdn"]
# Collections
zones = db["zones"]
nodes = db["nodes"]
users = db["users"]
ecas = db["ecas"]
zones.create_index([("zone", ASCENDING)], unique=True)
argon = PasswordHasher()
queue = BeanstalkClient("localhost", 11300)
with open("templates/zone.j2") as zone_template_file:
# noinspection JinjaAutoinspect
zone_template = Template(zone_template_file.read())
with open("templates/new_domain.j2", "r") as new_domain_template_file:
# noinspection JinjaAutoinspect
new_domain_template = Template(new_domain_template_file.read())
with open("templates/proxied_record.j2", "r") as proxied_record_template_file:
# noinspection JinjaAutoinspect
proxied_record_template = Template(proxied_record_template_file.read())
with open("templates/welcome.j2", "r") as welcome_template_file:
# noinspection JinjaAutoinspect
print("No jobs Available")
job=""
if (job!=""):
print(job.job_data)
executejob(job)
btclient.delete_job(job.job_id)
print("Deleting Command")
if __name__ == "__main__":
global btclient
btclient = BeanstalkClient('127.0.0.1', 11300)
while(True):
cycle()
from email.mime.text import MIMEText
from smtplib import SMTP_SSL as SMTP
from paramiko import SSHClient, AutoAddPolicy
from paramiko.ssh_exception import NoValidConnectionsError
from pymongo import MongoClient
from pystalk import BeanstalkClient
from scp import SCPClient
import utils
from config import configuration
db_client = MongoClient("mongodb://*****:*****@", "_").replace(".", "_")
def run_ssh_command(command):
print(" - running " + command, end="", flush=True)
stdin, stdout, stderr = ssh.exec_command(command)
def print_stats(server, port, tube):
client = BeanstalkClient(server, port)
tube_stats = client.stats_tube(tube)
print(" ready: {:d}\n buried: {:d}\ndelayed: {:d}".format(
tube_stats['current-jobs-ready'], tube_stats['current-jobs-buried'],
tube_stats['current-jobs-delayed']))