forked from brooklynne/utter-va
-
Notifications
You must be signed in to change notification settings - Fork 0
/
manage.py
executable file
·786 lines (630 loc) · 19.9 KB
/
manage.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
#!/usr/bin/env python
# -*- encoding:utf-8 -*-
# manage.py
import os
import sys
import time
import re
import gevent.monkey; gevent.monkey.patch_thread()
from IPy import IP
from flask import Flask
from flaskext.actions import Manager
from sqlalchemy import or_
from webapp import app, socketio, db
from webapp.models.models import User, Appliance, OpenStack, Status
from webapp.models.twitter import TwitterBot, TweetCommands
from webapp.models.images import Images
from webapp.models.flavors import Flavors
from webapp.models.instances import Instances
from webapp.models.addresses import Addresses
from webapp.libs.utils import query_yes_no, pprinttable, message
from webapp.libs.coinbase import coinbase_get_addresses, coinbase_checker
from webapp.libs.pool import pool_salesman, pool_connect
from webapp.libs import task_lock
# configuration file
if os.path.isfile('./DEV'):
app.config.from_object('config.DebugConfiguration')
else:
app.config.from_object('config.BaseConfiguration')
# manager handler
manager = Manager(app, default_help_actions=False)
# user, what to do?
def configure_blurb():
# get the appliance configuration
appliance = db.session.query(Appliance).first()
print "Visit http://%s/ to setup your appliance." % appliance.local_ip
# USERLAND METHODS
# reset the entire system
default_ip = "0.0.0.0"
def reset(app):
def action(ip=('i', default_ip)):
"""
Restores the appliance to factory default settings.
"""
try:
if ip == default_ip:
print "Please enter the appliance's IP address."
print "Usage: ./manage.py reset -i x.x.x.x"
return action
# double check they want to do this
if query_yes_no("Are you sure you want to reset the appliance?"):
# initialize database
path = os.path.dirname(os.path.abspath(__file__))
os.system('cp "%s/utterio.db" "%s/utterio_backup.db"' % (path, path))
# delete, then create all tables
db.drop_all()
db.create_all()
# initialize the appliance object
appliance = Appliance()
appliance.initialize(ip)
# sync with pool database
flavors = Flavors().sync()
if flavors['response'] != "success":
print flavors['result']
else:
print "The database has been cleared and a new API token has been generated."
configure_blurb()
except ValueError as ex:
print ex
return action
# reset the admin account
def admin(app):
def action(force=('f', 'false')):
"""
Resets the admin credentials. Run this, then access
the appliance's web page to create a new admin account.
"""
if force == 'true':
try:
user = db.session.query(User).first()
user.delete(user)
print "The admin user has been deleted. Please access the UI as soon as possible to create a new user."
configure_blurb()
except:
print "Appliance currently has no users."
configure_blurb()
else:
print "Doing nothing."
return action
# install
def install(app):
def action(ip=('i', default_ip)):
"""
Installs a new database configuration for the appliance.
"""
# create all tables
db.create_all()
if not Appliance.get():
# initialize the appliance object
appliance = Appliance()
appliance.initialize(ip)
# sync flavors from pool (openstack sync comes later when we have a user)
flavors = Flavors().sync()
# configure output
configure_blurb()
return action
# DEVELOPMENT METHODS
# serve application for development (in production we start it from monit)
def serve(app):
def action(gunicorn=('g', 'false')):
"""
Starts the development server.
"""
if gunicorn == 'true':
path = os.path.dirname(os.path.abspath(__file__))
# start gunicorn for development
os.system('gunicorn --max-requests 1 --access-logfile ./logs/access-log --error-logfile ./logs/error-log -c gunicorn.conf.py webapp:app')
sys.exit()
else:
socketio.run(app, host=default_ip)
return action
# coinop command
def coinop(app):
def action(
amount=('a', 0),
instance=('i', "smi-a4t0zcoe")
):
"""
Makes fake Bitcoin payments to the appliance. Example usage
paying instance 'ami-a4t0zcoe' 20 micro Bitcoin:
./manage.py coinop -a 20 -i ami-a4t0zcoe
"""
instance = db.session.query(Instances).filter_by(name=iname).first()
if amount == 0:
print "Enter a whole amount to pay instance."
elif instance:
print "Paying %s mBTC to instance %s." % (amount, iname)
instance.coinop(amount)
else:
print "Can't find that instance!"
return action
# show ips of running instances
def ips(app):
def action():
"""
Prints a list of current instances and their IP addresses.
"""
from webapp.libs.openstack import instance_info
# check appliance is ready to go - exit if not
settings = Status().check_settings()
if not settings['ngrok'] or not settings['openstack']:
print "Appliance is not ready."
return action
instances = db.session.query(Instances).filter_by(state=4).all()
if not instances:
print "No instances are running.\n"
for instance in instances:
print "Instance %s" % instance.name
print "====================="
# get instance (server) info
response = instance_info(instance)
server = response['result']['server']
for key in server.networks.keys():
for network in server.networks[key]:
print "%s IPv%s: %s" % (IP(network).iptype(), IP(network).version(), network)
# line break
print
return action
# quick and dirty openstack stats
def stats(app):
def action():
"""
Prints current hypervisor usage information.
"""
from webapp.libs.openstack import get_stats
# check appliance is ready to go - exit if not
settings = Status().check_settings()
if not settings['ngrok'] or not settings['openstack']:
print "Appliance is not ready."
return action
stats = get_stats()
print stats['result']['message']
for key in stats['result']['stats']:
print
print key
print stats['result']['stats'][key]
return action
# message client
def messenger(app):
def action(
text=('m', 'Hello from the administration console!'),
status=('s', 'success'),
reloader=('r', '0')
):
"""
Send messages and reload request to a browser connected to the appliance.
"""
# muck the reload flags around
if reloader == '1' or reloader.lower() == 'true':
reloader = True
else:
reloader = False
# send out the message
message(text, status, reloader)
return action
# build the tunnel.conf file
def tunnel(app):
def action():
"""
Builds a new Ngrok tunnel configuration file.
"""
# get the appliance configuration
appliance = db.session.query(Appliance).first()
if appliance.ngroktoken:
appliance.build_tunnel_conf()
else:
configure_blurb()
return action
# check authorization
def checkauth(app):
def action():
"""
Verify authentication token is being accepted by pool.
"""
# get the appliance configuration
appliance = db.session.query(Appliance).first()
if appliance.apitoken:
response = pool_connect("authorization", appliance)
print response['result']['message']
else:
configure_blurb()
return action
# CRONTAB METHODS
# grab the pool server's flavors and install
# runs every 15 minutes via cron
def flavors(app):
def action():
"""
Performs a sync from the pool's list of flavors to the appliance.
Cron: Every 15 minutes.
"""
# get the appliance for api token (not required, but sent if we have it)
appliance = db.session.query(Appliance).first()
# sync the flavors
flavors = Flavors()
flavors.sync()
flavors.sync_from_openstack(appliance)
return action
# grab the pool server's images and download
# runs every 15 minutes via cron
def images(app):
def action():
"""
Cleans up images that aren't used anymore
Cron: Every 15 minutes.
"""
# clear out old dynamic images
images = Images.get_all()
for image in images:
image.housekeeping()
return action
# cleans up decomissioned and errant instances
# runs every 15 minutes via cron
def trashman(app):
def action():
"""
Removes decomissioned and errant instances. Also takes out the trash occasionally.
Cron: Every 15 minutes.
"""
# check appliance is ready to go - exit if not
settings = Status().check_settings()
if not settings['ngrok'] or not settings['openstack']:
app.logger.error("Running trashman - appliance is not ready.")
return action
instances = db.session.query(Instances).filter_by(state=7).all()
for instance in instances:
response = instance.trashman()
message(response['result']['message'], "success", True)
return action
# salesman puts up instances for sale on pool
# runs every 15 minutes via cron
def salesman(app):
def action():
"""
Puts instances up for sale on the pool.
Cron: Every 15 minutes.
"""
# check appliance is ready to go - exit if not
settings = Status().check_settings()
if not settings['ngrok'] or not settings['openstack']:
app.logger.error("Running salesman - appliance is not ready.")
return action
# get the appliance
appliance = db.session.query(Appliance).first()
# instances for sale, get 'em while they're hot
instances = db.session.query(Instances).filter_by(state=1).all()
# call the pool with instances for sale
response = pool_salesman(instances, appliance)
return action
# mix, pause, unpause instances - remove old dynamic images
# runs every 5 minutes via cron
def housekeeper(app):
def action():
"""
Provides housekeeping services including mix, pause and decomission.
Cron: Every 5 minutes.
"""
# check appliance is ready to go - exit if not
settings = Status().check_settings()
if not settings['ngrok'] or not settings['openstack']:
app.logger.error("Running housekeeper - appliance is not ready.")
return action
# MIXING
# make sure we have mixed an instance for each flavor
instances = Instances()
flavors = Flavors.query.filter_by(installed=True, active=True).all()
# loop through the flavors we have and mix an instance
for flavor in flavors:
response = instances.mix(flavor)
if response['response'] != "success":
message("Instance mixing failed. Something is wrong.")
# HOUSEKEEPING
# general houskeeping work including pausing, unpausing
# runs on all currently running and suspended instances
instances = db.session.query(Instances).filter(or_(
Instances.state == 2,
Instances.state == 4,
Instances.state == 5
)).all()
# loop through them and do housekeeping
for instance in instances:
response = instance.housekeeping()
# if instance isn't running
if response['response'] == "error":
message(response['result']['message'], "error", True)
else:
if response['result']['message'] != "":
message(response['result']['message'], "success", True)
return action
# warmup and start instances
# runs every minute via cron
def instances(app):
def task():
# START
# instances which have received payment are moved to starting
instances = db.session.query(Instances).filter_by(state=2).all()
for instance in instances:
response = instance.start()
if response['response'] == "success":
message("Instance %s launched." % instance.name, "success", True)
elif response['response'] == "queued":
message("Instance %s is waiting for image." % instance.name, "success", True)
else:
message("%s Unable to launch instance %s." % (
response['result']['message'],
instance.name
),
"error",
True
)
instance.message = response['result']['message']
instance.update()
# NUDGE
# instances in the process of starting are monitored and updated
instances = db.session.query(Instances).filter_by(state=3).all()
for instance in instances:
response = instance.nudge()
if response['response'] == "success":
message("Instance %s is now running." % instance.name, "success", True)
# RELIGHT
# instances which have unpaused via payment
instances = db.session.query(Instances).filter_by(state=6).all()
# loop through them and do housekeeping
for instance in instances:
response = instance.housekeeping()
# if instance isn't running
if response['response'] == "error":
message(response['result']['message'], "error", True)
instance.message = response['result']['message']
instance.update()
else:
if response['result']['message'] != "":
message(response['result']['message'], "success", True)
return
def action(
cron=('c', 0),
freq=('f', 0),
):
"""
Provides instance services including start, nudge and relight.
Cron: Every 1 minute.
"""
# check appliance is ready to go - exit if not
settings = Status().check_settings()
if not settings['ngrok'] or not settings['openstack']:
app.logger.error("Running instances - appliance is not ready.")
return action
# check flags for non-cron run (for dev)
if cron == 0 or freq == 0:
task()
return action
# current UTC time in seconds since epoch
epoch_time = int(time.time())
# cron, frequency length in seconds, run_time
cron_frequency = cron
frequency = freq
run_time = 0
# do a single run
timer_in = int(time.time())
task()
timer_out = int(time.time())
# run task X many more times per cron period
for x in range(1,cron_frequency/frequency):
# run time
run_time = timer_out - timer_in
# sleep for a a bit
if run_time < frequency:
time.sleep(frequency - run_time)
# check if we are going to go over on next run
est_time = (frequency * x) + run_time
if est_time > cron_frequency:
break
# wrap task above with time in and out
timer_in = int(time.time())
task()
timer_out = int(time.time())
return action
# twitter stream + db storage
# uses a forever BLOCKING call to get_stream()
# runs from monit
def tweetstream(app):
def action():
"""
Stream process for Twitter monitoring. Do not run directly.
"""
from webapp.libs.twitterbot import get_stream
# bot settings
bot = TwitterBot.get()
if bot:
if bot.enabled:
get_stream()
else:
# we're going to exit, and monit will try to restart
# delay that a bit so monit doesn't freak out
time.sleep(60)
else:
time.sleep(60)
return action
# handle the request queue from the twitter stream process
def falconer(app):
def task(bot):
from webapp.libs.twitterbot import tweet_status, run_status, reserve_instance, check_instance, cleanup_reservations
# get unhandled commands
commands = db.session.query(TweetCommands).filter_by(state=1).all()
for command in commands:
# someone typed '!instance'
if command.command == "instance":
# reserve instance
response = reserve_instance(command, bot)
if response['response'] == "error":
print response['result']['message']
command.delete(command)
# someone typed '!status'
elif command.command.lower() == "status":
# send status info
response = run_status(command, bot)
if response['response'] == "error":
print response['result']
# don't hold onto status commands
command.delete(command)
elif command.command.lower() == "help":
response = tweet_status("'@%s !instance ^http://pastebin⋅com/raw.php?i=zX5fD6HY' & pay. Edit pastebin to suit! Also, '@%s !status'." % (bot.screen_name, bot.screen_name), command.user)
if response['response'] == "error":
print response['result']
command.delete(command)
else:
# some other command or errant tweet
command.delete(command)
# update status of commands carrying an instance_id and update
commands = db.session.query(TweetCommands).filter_by().all()
for command in commands:
# run an instance check
check_instance(command, bot)
# get pending reserved instance commands
commands = db.session.query(TweetCommands).filter_by(state=10).all()
for command in commands:
cleanup_reservations(command, bot)
return
def action(
cron=('c', 0),
freq=('f', 0),
):
"""
Provides Twitter command processing.
Cron: Every 1 minute.
"""
# get bot settings
bot = TwitterBot.get()
# exit if we're not enabled
if not bot:
return action
if not bot.enabled:
print "The marketing bot is disabled."
return action
# check flags for non-cron run (for dev)
if cron == 0 or freq == 0:
task(bot)
return action
# current UTC time in seconds since epoch
epoch_time = int(time.time())
# cron, frequency length in seconds, run_time
cron_frequency = cron
frequency = freq
run_time = 0
# do a single run
timer_in = int(time.time())
task(bot)
timer_out = int(time.time())
# run task X many more times per cron period
for x in range(1,cron_frequency/frequency):
# run time
run_time = timer_out - timer_in
# sleep for a a bit
if run_time < frequency:
time.sleep(frequency - run_time)
# check if we are going to go over on next run
est_time = (frequency * x) + run_time
if est_time > cron_frequency:
break
# wrap task above with time in and out
timer_in = int(time.time())
task(bot)
timer_out = int(time.time())
return action
# advertising agent
# announce instances
def marketeer(app):
def action():
"""
Posts marketing blurbs to Twitter.
Cron: Depends on offering period.
"""
import random
from webapp.libs.twitterbot import tweet_status
# get bot settings
bot = TwitterBot.get()
# get the time
epoch = int(time.time())
if not bot:
print "The marketing bot is disabled."
return action
if bot.announce == 0:
print "The marketing bot is disabled."
return action
# if updated + announce > current time, do an update!
if epoch > (bot.updated + (bot.announce * 3600)):
# make up some stuff
blurbs = [
"Get your hot-n-fresh #openstack instances! ",
"Instances for nothing and #bitcoins for free. ",
"Now serving #42. ",
"Pssst. Hey buddy, want some #openstack? ",
"Sorry #openstack, we're all out of IPv4 addresses. ",
"Any significantly advanced technology is indistinguishable from magic. ",
"It's #bitcoin magic! ",
"I'm hungry. Spare some #bitcoin? "
]
hashtags = [
"trust",
"cryptocurrency",
"transparency",
"globalcloud",
"federation",
"virtualization",
"monkeys"
]
blurb = random.choice(blurbs)
hashtag = random.choice(hashtags)
# say it
tweet_status("%s '@%s !status' for more info. #%s" % (blurb, bot.screen_name, hashtag))
# update
bot.updated = int(time.time())
bot.update()
return action
# http://pastebin.com/raw.php?i=zX5fD6HY
# for development
manager.add_action('serve', serve)
manager.add_action('coinop', coinop)
manager.add_action('message', messenger)
manager.add_action('ips', ips)
manager.add_action('stats', stats)
manager.add_action('admin', admin)
# commands for user managment
manager.add_action('reset', reset)
manager.add_action('install', install)
manager.add_action('tunnel', tunnel)
manager.add_action('checkauth', checkauth)
# run from cron every 15 minutes
manager.add_action('images', images)
manager.add_action('flavors', flavors)
manager.add_action('trashman', trashman)
manager.add_action('salesman', salesman)
manager.add_action('marketeer', marketeer)
# run from cron every 5 mintues
manager.add_action('housekeeper', housekeeper)
# run from cron every minute
manager.add_action('instances', instances)
manager.add_action('falconer', falconer)
# twitter commands run from monit in prod
manager.add_action('tweetstream', tweetstream)
if __name__ == "__main__":
# manager logs
import logging
from logging.handlers import RotatingFileHandler
# delete existing handlers
del app.logger.handlers[:]
handler = RotatingFileHandler('%s/logs/utter.log' % os.path.dirname(os.path.realpath(__file__)), maxBytes=1000000, backupCount=7)
handler.setLevel(logging.INFO)
log_format = "%(asctime)s - %(levelname)s - %(message)s"
formatter = logging.Formatter(log_format)
handler.setFormatter(formatter)
app.logger.addHandler(handler)
# deal with glance client logs
logging.getLogger('glanceclient.common.http').addHandler(handler)
# lock things down so we don't run over the top of another run
lock_config = {'lock_dir': app.config['LOCK_DIRECTORY']}
if len(sys.argv) > 0:
lock_config['lock_name'] = sys.argv[1]
else:
lock_config['lock_name'] = 'anonymous'
task_lock.LockManager(**lock_config).run_once(manager.run)