def storage_manager_runner(sm_config, smtp, cluster_name, atomic_ref):
while True:
try:
sm = StorageManager(sm_config, smtp, cluster_name, atomic_ref)
sm.run()
except:
logger.exception("storage_manager_runner failed")
def main():
args = CliParser().create_parser().parse_args()
storage_manager = StorageManager()
content = storage_manager.getContent(args.source, args.input_encoding)
content = Converter().convert(content)
if not args.dry_run:
dest = args.dest
if dest == "":
dest = get_default_output_name(args.source)
storage_manager.saveFile(content,dest, args.output_encoding)
if args.screen:
print(content)
def main():
while True:
sm_config = config.get("storage_manager", None)
if sm_config is None:
logger.info("storage_manager is not enabled. Exiting ...")
sys.exit(0)
smtp = config.get("smtp", None)
cluster_name = config.get("cluster_name_friendly", None)
sm = StorageManager(sm_config, smtp, cluster_name)
try:
sm.run()
except Exception as e:
logger.error("StorageManager.run failed with exception %s", e)
def annotationSchema(schema_id):
"""
:route: **/annotationSchema/<schema_id>**
Used to store annotation schemas.
Same implementation as for the document.
"""
man = StorageManager()
try:
man.setCollection(
settings.GetConfigValue("ServiceStockageAnnotations",
"SchemaCollection"))
man.connect()
if request.method == 'GET':
logger.logUnknownDebug("Get Schema", "Id: {0}".format(schema_id))
doc = man.getMongoDocument(schema_id)
_convStorageIdToDocId(doc)
if (doc is None):
raise (StorageRestExceptions(2))
else:
return jsonify(doc)
elif request.method == 'PUT':
doc = request.json
_convDocIdToStorageId(doc)
if '_id' in doc and doc["_id"] != schema_id:
raise (StorageRestExceptions(1))
else:
logger.logUnknownDebug("Update Schema",
"Id: {0}".format(schema_id))
docId = man.updateMongoDocument(request.json)
if (docId is None):
raise (StorageRestExceptions(3))
return jsonify({"id": docId})
elif request.method == 'DELETE':
logger.logUnknownDebug("Delete Schema",
"Id: {0}".format(schema_id))
man.deleteMongoDocument(schema_id)
# Whenever it is true or false we don't care, if there is no
# exception
return jsonify({}), 204
else:
return error_response(httplib.BAD_REQUEST, "Bad Request", "", "")
except Exception, e:
return _processCommonException(e)
def on_parent(self, widget, parent):
print("on_parent event -> full layout available")
pizzas_json = StorageManager().get_data(
self.STORAGE_FILENAME) # get pizzas from file
self.recycleView.data = Pizza.build_list_data_for_rw(pizzas_json)
HttpClient().get_pizzas(self.on_server_data, self.on_error_data,
self.on_failure_data, self.on_progress_data)
print(
"HttpClient().get_pizzas is called to get pizzas from web service")
def __init__(self):
global main
super().__init__()
main = self
self.setupUi(self)
self.submit.clicked.connect(self.on_input)
self.data_edit.returnPressed.connect(self.submit.click)
self.storage_manager = StorageManager()
self.storage_manager.start()
self.music_player = MusicPlayer()
self.music_player.start()
self.scheduler = Scheduler(self)
self.scheduler.start()
self.log_units = [
self.storage_manager, self.music_player, self.scheduler
]
self.checkbox_list = []
for k in range(16):
self.checkbox_list.append(getattr(self, f'checkbox_list_item{k}'))
def get_checkbox_handler(checkbox, k, callback):
return lambda: callback(checkbox, k)
for k, log_unit in enumerate(self.log_units):
self.checkbox_list[k].setText(log_unit.log_header)
self.checkbox_list[k].setChecked(True)
self.checkbox_list[k].toggled.connect(
get_checkbox_handler(self.checkbox_list[k], k,
self.on_toggle_checkbox))
self.checkbox_list_autoscroll.setChecked(True)
self.console_log = QStringListModel()
self.console.setModel(self.console_log)
self.schedule_tree.setHeaderLabels(['일자', '시간', '행동'])
for head in entire_schedule:
header = QTreeWidgetItem([head])
for tail_time in entire_schedule[head]:
header.addChild(
QTreeWidgetItem([
None,
str(tail_time), entire_schedule[head][tail_time]
]))
self.schedule_tree.addTopLevelItem(header)
header.setExpanded(True)
self.schedule_tree.doubleClicked.connect(
self.on_double_click_schedule_tree)
for k, macro_name in enumerate(quick_macros):
create_quick_macro_button(k, macro_name, quick_macros[macro_name])
self.log_timer = QTimer()
self.log_timer.setInterval(1000)
self.log_timer.timeout.connect(self.on_check_log)
self.log_timer.start()
self.show()
class MainWindow(QtWidgets.QMainWindow, uic.loadUiType('data/planner.ui')[0]):
def __init__(self):
global main
super().__init__()
main = self
self.setupUi(self)
self.submit.clicked.connect(self.on_input)
self.data_edit.returnPressed.connect(self.submit.click)
self.storage_manager = StorageManager()
self.storage_manager.start()
self.music_player = MusicPlayer()
self.music_player.start()
self.scheduler = Scheduler(self)
self.scheduler.start()
self.log_units = [
self.storage_manager, self.music_player, self.scheduler
]
self.checkbox_list = []
for k in range(16):
self.checkbox_list.append(getattr(self, f'checkbox_list_item{k}'))
def get_checkbox_handler(checkbox, k, callback):
return lambda: callback(checkbox, k)
for k, log_unit in enumerate(self.log_units):
self.checkbox_list[k].setText(log_unit.log_header)
self.checkbox_list[k].setChecked(True)
self.checkbox_list[k].toggled.connect(
get_checkbox_handler(self.checkbox_list[k], k,
self.on_toggle_checkbox))
self.checkbox_list_autoscroll.setChecked(True)
self.console_log = QStringListModel()
self.console.setModel(self.console_log)
self.schedule_tree.setHeaderLabels(['일자', '시간', '행동'])
for head in entire_schedule:
header = QTreeWidgetItem([head])
for tail_time in entire_schedule[head]:
header.addChild(
QTreeWidgetItem([
None,
str(tail_time), entire_schedule[head][tail_time]
]))
self.schedule_tree.addTopLevelItem(header)
header.setExpanded(True)
self.schedule_tree.doubleClicked.connect(
self.on_double_click_schedule_tree)
for k, macro_name in enumerate(quick_macros):
create_quick_macro_button(k, macro_name, quick_macros[macro_name])
self.log_timer = QTimer()
self.log_timer.setInterval(1000)
self.log_timer.timeout.connect(self.on_check_log)
self.log_timer.start()
self.show()
def closeEvent(self, c):
print('Closing...')
self.log_timer.stop()
for log_unit in self.log_units:
log_unit.close()
log_unit.quit()
def insert_log(self, log: str):
row = self.console_log.rowCount()
self.console_log.insertRow(row)
index = self.console_log.index(row)
print(log)
self.console_log.setData(index, log)
@pyqtSlot(QModelIndex)
def on_double_click_schedule_tree(self, index):
try:
key = index.model().data(index, 2)
if key is not None:
self.parse_command(f'-{key}')
except:
self.insert_log(traceback.format_exc())
@pyqtSlot()
def on_toggle_checkbox(self, check_box, k):
if k < len(self.log_units):
self.log_units[k].log_enable = check_box.isChecked()
@pyqtSlot()
def on_input(self):
text = self.data_edit.text()
self.parse_command(text.lower())
self.console.scrollToBottom()
self.data_edit.setText('')
for k, method_name in enumerate(quick_macros):
button = getattr(self, f'button_list_item{k}')
print(button.x(), button.y())
@pyqtSlot()
def on_check_log(self):
logs_to_insert = []
for k, log_unit in enumerate(self.log_units):
if log_unit.is_empty():
continue
for log in log_unit.flush_log():
logs_to_insert.append(log)
if len(logs_to_insert) > 0:
logs_to_insert.sort(key=lambda x: x[0])
for log in logs_to_insert:
self.insert_log('%02d:%02d: %s' %
(log[0].hour, log[0].minute, log[1].replace(
os.getcwd(), '$://')))
if self.checkbox_list_autoscroll.isChecked():
self.console.scrollToBottom()
@pyqtSlot(str)
def parse_command(self, command: str):
self.insert_log(f'>> {command}')
try:
if command == 'test':
self.music_player.music_pause()
self.music_player.play_unstoppable_music('audio/테스트.mp3')
self.music_player.music_resume()
elif command == 'get_musics':
with self.storage_manager.lock:
log = '=================현재 기상송===============\n'
for file_path in self.storage_manager.files_to_play:
log += file_path + '\n'
log += '========================================='
self.insert_log(log)
elif command == 'all_around_test':
self.scheduler.all_around_test()
elif command == 'abort_song':
self.music_player.close()
elif command.startswith('-'):
self.scheduler.push_tag_command(command[1:])
else:
self.insert_log(f'?? {command}')
except:
self.insert_log(traceback.format_exc())
self.insert_log(f'<< {command}')
def on_server_data(self, data):
print("on_server_data...")
self.recycleView.data = Pizza.build_list_data_for_rw(data)
StorageManager().store_data(self.STORAGE_FILENAME, data)
from IPython import embed
import time
from storage_manager import StorageManager, RedisManager, RedisDetailQueue
sm = StorageManager()
new_proxy = sm.create_proxy('1.1.1.3', 8080, 'http')
new_proxy = sm.create_proxy('1.1.1.4', 8080, 'http')
new_proxy = sm.create_proxy('1.1.1.5', 8080, 'http')
rdq1 = RedisDetailQueue(queue_key='q_1', active=False)
tq1 = sm.create_queue('https://www.google.com')
tq2 = sm.create_queue('https://www.bing.com')
rdq2 = RedisDetailQueue(queue_key=tq1.queue_key)
for i in range(rdq1.length()):
detail_to_clone = rdq1.dequeue()
clone = sm.clone_detail(detail_to_clone, tq1)
clone = sm.clone_detail(detail_to_clone, tq2)
sm.sync_to_db()
embed()
def cli():
"""dss4gcp supports the setup of gcp infrastrucutre"""
global storage, compute, service_accounts
storage = StorageManager()
compute = ComputeManager(project='dataiku-cluster', zone='us-central1-a')
service_accounts = ServiceAccountManager(project='dataiku-cluster')
def __init__(self):
self.storage_mgr = StorageManager()
class ProxyManager(object):
def __init__(self):
self.storage_mgr = StorageManager()
def load_from_seed_queue(self, queue, num=None):
if queue.queue_id == SEED_QUEUE_ID:
return
seed_queue = self.storage_mgr.redis_mgr.get_queue_by_id(SEED_QUEUE_ID)
seed_rdq_active = RedisDetailQueue(queue_key=seed_queue.queue_key,
active=True)
seed_rdq_inactive = RedisDetailQueue(queue_key=seed_queue.queue_key,
active=False)
active_seeds_to_dequeue = min(ACTIVE_PROXIES_PER_QUEUE,
seed_rdq_active.length())
inactive_seeds_to_dequeue = INACTIVE_PROXIES_PER_QUEUE - active_seeds_to_dequeue
if num is not None:
active_seeds_to_dequeue = num
inactive_seeds_to_dequeue = num
for i in range(active_seeds_to_dequeue):
try:
self.storage_mgr.clone_detail(seed_rdq_active.dequeue(), queue)
except Exception as e:
pass
for i in range(inactive_seeds_to_dequeue):
self.storage_mgr.clone_detail(seed_rdq_inactive.dequeue(), queue)
def get_seed_proxy(self, queue):
if queue.id == SEED_QUEUE_ID:
logging.warn("trying to copy seed proxy to seed_queue")
return
seed_queue = self.storage_mgr.redis_mgr.get_queue_by_id(SEED_QUEUE_ID)
self.storage_mgr.clone_detail(seed_queue.dequeue(), queue)
def get_proxy(self, request_url):
domain = parse_domain(request_url)
queue = self.storage_mgr.redis_mgr.get_queue_by_domain(domain)
rdq_active = RedisDetailQueue(queue_key=queue.queue_key, active=True)
rdq_inactive = RedisDetailQueue(queue_key=queue.queue_key,
active=False)
logging.info("active queue count: %s" % rdq_active.length())
logging.info("inactive queue count: %s" % rdq_inactive.length())
use_active = True
clone_seed = flip_coin(SEED_FREQUENCY)
if rdq_active.length() < MIN_ACTIVE:
self.load_from_seed_queue(queue)
elif clone_seed:
self.load_from_seed_queue(queue, num=1)
if rdq_active.length() < MIN_ACTIVE:
use_active = False
if flip_coin(INACTIVE_PCT):
use_active = False
if use_active and rdq_active.length() > 0:
logging.info("using active queue")
draw_queue = rdq_active
else:
logging.info("using inactive queue")
draw_queue = rdq_inactive
detail = None
detail = draw_queue.dequeue(requeue=False)
proxy = ProxyObject(self.storage_mgr, detail)
proxy.dispatch(rdq_active, rdq_inactive)
return proxy
def on_server_data(self, pizzas_dict):
self.recycleView.data = pizzas_dict
StorageManager().save_data("pizzas", pizzas_dict)
def on_parent(self, widget, parent):
# l = [pizza.get_dictionary() for pizza in self.pizzas]
pizzas_dict = StorageManager().load_data("pizzas")
if pizzas_dict:
self.recycleView.data = pizzas_dict