def load_all_account_configurations(self):
logger.debug('Loading account configurations from {}'.format(
self.__account_configuration_file))
if not os.path.isfile(self.__account_configuration_file):
raise FileNotFoundError('{} does not exist'.format(
self.__account_configuration_file))
with open(self.__account_configuration_file, 'r') as f:
configured_accounts = yaml.round_trip_load(f, preserve_quotes=True)
for key in configured_accounts.keys():
if key == 'version':
continue
try:
configured_accounts[key]['password'] = keyring.get_password(
AccountManager.KEYRING_APPLICATION_IDENTIFIER, key)
self.__cache[key] = Account.from_dict(key,
configured_accounts[key])
Environment.bootstrap_account_directories(self.__cache[key])
except KeyError as e:
logger.warning(e, exc_info=True)
def __init__(self):
Environment.bootstrap_application_directories()
self._app = QApplication(sys.argv)
self._app.setStyle(QStyleFactory.create("Fusion"))
self._app.setWindowIcon(
QtGui.QIcon(
os.path.join(Environment.get_resource_path(),
'designs/default/OXNote.icns')))
self._app.setApplicationDisplayName('OXNote')
self._app.setApplicationName('OXNote')
def start(sys_args):
'''Start pipeline with sys_args'''
env = Environment()
# TODO : this parser needs a lot of work
parent_parser = argparse.ArgumentParser(prog="vertex_pipeline",
description='commands to manipulate vertex pipelines')
sub_parsers = parent_parser.add_subparsers(dest='subparser_name')
run_parser = sub_parsers.add_parser('run',
description='run a pipeline script')
run_parser.add_argument('path',
help='path to pipeline to run')
run_parser.add_argument('-e','--use_ec2',action='store_true',
help='use this option to run pipeline on an ec2 instance')
run_parser.add_argument('-s','--start_at',metavar=('s'),
help='set a start point (s) to begin running pipeline at')
run_parser.add_argument('pipeline_argument',nargs='*',
help='argument to pass to the pipeline')
set_parser = sub_parsers.add_parser('set',
description='set an environment variable')
set_parser.add_argument('name',
help='environment variable to set')
set_parser.add_argument('value',
help='value to set for environment variable')
get_parser = sub_parsers.add_parser('get',
description='get an environment variable')
get_parser.add_argument('name',
help='environment variable to set')
showAll_parser = sub_parsers.add_parser('showAll',
description='show all environment variables and their values')
vals = vars(parent_parser.parse_args([sys_args[i] for i in range(1,len(sys_args))]))
# TODO : need to test, not using start_at yet
if vals['subparser_name'] == 'run':
runner = PipelineRunner(vals['path'])
if vals['use_ec2']:
runner.runPipelineOnEc2(pipeline_args=vals['pipeline_argument'])
else:
runner.runPipeline(pipeline_args=vals['pipeline_argument'])
elif vals['subparser_name'] == 'set':
env.set(vals['name'],vals['value'])
elif vals['subparser_name'] == 'get':
val = env.get(vals['name'])
if val is not None:
print(val)
elif vals['subparser_name'] == 'showAll':
env.showAll()
env.save()
def main():
logger.debug('Initiated logging environment')
Environment.initiate_logging_environment(Configuration().get_setting(
'oxnote', 'extended_logging.requests_debug_level', default=0))
logger.debug('Starting OXNote')
global application
application = Application()
if not AccountManager().list_accounts():
application.start_wizard()
else:
application.start_main_window()
def __load_fonts(self):
qta.load_font('far',
'fa-regular-400.ttf',
'charmap.json',
directory=os.path.join(Environment.get_resource_path(),
'fonts'))
qta.load_font('fas',
'fa-solid-900.ttf',
'charmap.json',
directory=os.path.join(Environment.get_resource_path(),
'fonts'))
qta.load_font('fab',
'fa-brands-400.ttf',
'charmap.json',
directory=os.path.join(Environment.get_resource_path(),
'fonts'))
def __set_icons(self):
if not QtWidgets.QSystemTrayIcon.isSystemTrayAvailable():
logger.error('System Tray is not accessible')
else:
self.tray_icon = QSystemTrayIcon(
QIcon(
os.path.join(Environment.get_resource_path(),
'designs/default/OXNoteTray.icns')))
self.tray_icon.show()
self.tool_button_toolbar_top_sync.setIcon(
qta.icon('fa.refresh', color='lightgrey'))
self.tool_button_toolbar_left_notes.setIcon(
qta.icon('fa.clipboard', color='lightgrey'))
self.tool_button_toolbar_left_trash.setIcon(
qta.icon('fa.trash', color='lightgrey'))
self.tool_button_toolbar_left_shared.setIcon(
qta.icon('fa.share-alt', color='lightgrey'))
self.tool_button_toolbar_left_toggle.setIcon(
qta.icon('fa.angle-double-left', color='lightgrey'))
self.tool_button_editor_bold.setIcon(qta.icon('fa.bold'))
self.tool_button_editor_italic.setIcon(qta.icon('fa.italic'))
self.tool_button_editor_underline.setIcon(qta.icon('fa.underline'))
self.tool_button_editor_color.setIcon(qta.icon('fa.tint'))
self.tool_button_editor_bullet_list.setIcon(qta.icon('fa.list-ul'))
self.tool_button_editor_numbered_list.setIcon(qta.icon('fa.list-ol'))
self.tool_button_editor_outdent.setIcon(qta.icon('fa.outdent'))
self.tool_button_editor_indent.setIcon(qta.icon('fa.indent'))
self.tool_button_editor_tags.setIcon(qta.icon('fa.tags'))
self.tool_button_editor_sharing.setIcon(qta.icon('fa.cloud-upload'))
self.tool_button_editor_information.setIcon(qta.icon('fa.info'))
self.tool_button_editor_trash.setIcon(qta.icon('fas.trash-alt'))
def __init__(self, *args):
QWidget.__init__(self, *args)
loadUi(
os.path.join(Environment.get_resource_path(),
'designs/default/setup_wizard.ui'), self)
self.push_button_start.setEnabled(False)
def get_account_root_directory_path(self) -> (str, None):
if self._account_root_directory and self._oxnote_home_folder:
return os.path.join(
Environment.get_base_path(),
Configuration().get_setting('oxnote',
'application.directories.accounts',
default='.oxnote/accounts'),
self._account_root_directory)
def __init__(self, *args):
'''
..todo:: Remove unnecessary fonts.
..todo:: qtawesome seems to create lag during window resizing, investigate.
'''
QMainWindow.__init__(self, *args)
loadUi(
os.path.join(Environment.get_resource_path(),
'designs/default/main_window.ui'), self)
if os.path.isfile(
os.path.join(Environment.get_resource_path(),
'designs/default/main_window.qss')):
with open(
os.path.join(Environment.get_resource_path(),
'designs/default/main_window.qss'), "r") as f:
self.setStyleSheet(f.read())
self._account_manager = AccountManager()
self._synchronization_daemon = SynchronizationDaemon()
self._synchronization_daemon.signal_synchronization_started.connect(
self.notes_synchronization_initiated)
self._synchronization_daemon.signal_synchronization_status_update.connect(
self.statusbar_message)
self._synchronization_daemon.signal_synchronization_note_updated.connect(
self.signal_note_updated)
self._synchronization_daemon.signal_synchronization_finished.connect(
self.notes_synchronization_finished)
self._synchronization_daemon.enable()
self.__load_fonts()
self.__set_icons()
self.stacked_widget_detail.setCurrentIndex(
self.DETAIL_WIDGET_INDEX_NOTES)
self.__connect_signals()
self.__initialize_notes_list()
self._synchronization_daemon.synchronize()
def __init__(self, filename: str, checksum: str, parent=None):
super(NotesListWidgetItemWidget, self).__init__(parent)
self._filename = filename
self._checksum = checksum
loadUi(
os.path.join(Environment.get_resource_path(),
'designs/default/list_widget.ui'), self)
if os.path.isfile(
os.path.join(Environment.get_resource_path(),
'designs/default/list_widget.qss')):
with open(
os.path.join(Environment.get_resource_path(),
'designs/default/list_widget.qss'), "r") as f:
self.setStyleSheet(f.read())
self.label_preview_image.hide()
def insertFromMimeData(self,
source: QMimeData,
disable_richtext: bool = False):
'''
..todo: Add support for embedded content when inserting html mime from clipboard
'''
if source.hasImage():
temporary_file = os.path.join(
Environment.get_base_path(),
Configuration().get_setting(
'oxnote',
'application.directories.temporary',
default='.oxnote/tmp'), '{}.png'.format(
(str(uuid.uuid4()))))
source.imageData().save(temporary_file)
with open(temporary_file, 'rb') as f:
encoded = base64.b64encode(f.read())
self.textCursor().insertImage('data:image/png;base64,{}'.format(
encoded.decode("utf-8")))
if os.path.isfile(temporary_file):
os.remove(temporary_file)
elif source.hasUrls():
for url in source.urls():
if pathlib.Path(url.fileName()).suffix.lower(
)[1:] not in self.supported_image_formats:
super().insertFromMimeData(source)
continue
file_extension = pathlib.Path(
url.fileName()).suffix.lower()[1:]
if url.isLocalFile():
if not os.path.isfile(url.toLocalFile()):
continue
with open(url.toLocalFile(), 'rb') as f:
self.textCursor().insertImage(
'data:image/png;base64,{}'.format(
base64.b64encode(f.read()).decode("utf-8")))
else:
response = requests.get(url.toString(), stream=True)
if response.status_code == 200:
self.textCursor().insertImage(
'data:image/{};base64,{}'.format(
file_extension,
base64.b64encode(
response.content).decode("utf-8")))
elif source.hasHtml() and disable_richtext:
self.textCursor().insertText(source.text())
else:
super().insertFromMimeData(source)
def __init__(self):
self.RUN_ORDER = [] # ordered list of functions to run
self.outputDir = "" # pipeline output directory
self.logPath = "" # path to log file
self.logFile = None # log file object
self.env = Environment() # environment variables
self.name = "" # name of pipeline
self.desc = "" # description of pipeline
self.result = None # stores result from last RUN_ORDER function call
def __init__(self, aws_access_key=None, aws_secret_key=None, region=None):
'''Default constructor.
Inputs:
aws_access_key = access key provided by aws
aws_secret_key = secret key associated with access key'''
self.env = Environment()
if aws_access_key is None:
self.__access_key = self.env.get("ACCESS_KEY")
else:
self.__access_key = aws_access_key
if aws_secret_key is None:
self.__secret_key = self.env.get("SECRET_KEY")
else:
self.__secret_key = aws_secret_key
self.__region = region
self.__conn = EC2Connection(aws_access_key_id=self.__access_key,
aws_secret_access_key=self.__secret_key)
self.__runningInstances = {}
class SemanticVisitor(SemanticVisitorBase):
def __init__(self, warn_unhandled=False, visit_internal_types=True):
self.prefix = None
self._stack = []
self.env = Environment()
self.warn_unhandled = warn_unhandled
self.visit_internal_types = visit_internal_types
def get_stack(self):
return self._stack
stack = property(get_stack)
def stack_as_string(self):
return " > ".join([str(obj) for obj in self._stack])
# HANDLERS IMPLEMENTING VISITOR and adding a handle call
@stacked
@with_handling
def visit_Identifier(self, id):
pass
@stacked
@with_handling
def visit_Model(self, model):
for module in model.modules.values():
module.accept(self)
@stacked
@with_handling
def visit_Domain(self, domain):
self.env[domain.name] = domain
@stacked
@with_handling
def visit_Scope(self, scope):
pass
@stacked
@with_handling
def visit_Module(self, module):
self.env.extend()
for constant in module.constants:
constant.accept(self)
for name, function in module.externals.items():
self.env[name] = function
for domain in module.domains:
domain.accept(self)
for extension in domain.extensions:
extension.accept(self)
for function in module.functions:
# we only want to handle non-anonymous function declarations here
# because anonymous declarations are declared in their specific scope
# TODO: do this in a cleaner way (AnonFunctionDecl?)
if function.name[0:9] != "anonymous":
function.accept(self)
for execution in module.executions:
execution.accept(self)
self.env.reduce()
@stacked
@with_handling
def visit_Constant(self, constant):
self.env[constant.name] = constant
@stacked
@with_handling
def visit_Extension(self, extension):
extension.domain.accept(self)
extension.extension.accept(self)
@stacked
@with_handling
def visit_Every(self, execution):
execution.interval.accept(self)
execution.scope.accept(self)
execution.executed.accept(self)
@stacked
@with_handling
def visit_When(self, execution):
execution.scope.accept(self)
execution.event.accept(self)
execution.executed.accept(self)
@stacked
@with_handling
def visit_FunctionDecl(self, function):
self.env[function.name] = function
self.env.extend()
function.type.accept(self)
for param in function.parameters:
param.accept(self)
function.body.accept(self)
self.env.reduce()
@stacked
@with_handling
def visit_Parameter(self, parameter):
self.env[parameter.name] = parameter
parameter.type.accept(self)
# STATEMENTS
@stacked
@with_handling
def visit_BlockStmt(self, block):
self.env.extend()
for statement in block.statements:
statement.accept(self)
self.env.reduce()
@stacked
@with_handling
def visit_AssignStmt(self, stmt):
stmt.value.accept(self)
stmt.variable.accept(self)
@stacked
@with_handling
def visit_AddStmt(self, stmt):
stmt.variable.accept(self)
stmt.value.accept(self)
@stacked
@with_handling
def visit_SubStmt(self, stmt):
stmt.variable.accept(self)
stmt.value.accept(self)
@stacked
@with_handling
def visit_IncStmt(self, stmt):
stmt.variable.accept(self)
@stacked
@with_handling
def visit_DecStmt(self, stmt):
stmt.variable.accept(self)
@stacked
@with_handling
def visit_IfStmt(self, stmt):
stmt.condition.accept(self)
stmt.true.accept(self)
if stmt.false != None:
stmt.false.accept(self)
@stacked
@with_handling
def visit_CaseStmt(self, stmt):
stmt.expression.accept(self)
for case, consequence in zip(stmt.cases, stmt.consequences):
self.env.extend()
case.accept(self)
consequence.accept(self)
self.env.reduce()
@stacked
@with_handling
def visit_ReturnStmt(self, stmt):
if stmt.expression != None:
stmt.expression.accept(self)
# EXPRESSIONS
@stacked
@with_handling
def visit_BooleanLiteralExp(self, exp):
pass
@stacked
@with_handling
def visit_IntegerLiteralExp(self, exp):
pass
@stacked
@with_handling
def visit_FloatLiteralExp(self, exp):
pass
@stacked
@with_handling
def visit_AtomLiteralExp(self, atom):
pass
@stacked
@with_handling
def visit_UnknownType(self, lst):
pass
def visit_AnyType(self, type):
pass
def visit_MixedType(self, type):
pass
def visit_AtomType(self, type):
pass
@stacked
@with_handling
def visit_ListLiteralExp(self, lst):
if self.visit_internal_types: lst.type.accept(self)
for exp in lst.expressions:
exp.accept(self)
@stacked
@with_handling
def visit_ObjectLiteralExp(self, obj):
for prop in obj.properties:
prop.accept(self)
@stacked
@with_handling
def visit_Property(self, prop):
if prop.type != None:
prop.type.accept(self)
prop.value.accept(self)
# types
@stacked
@with_handling
def visit_VoidType(self, type):
pass
@stacked
@with_handling
def visit_BooleanType(self, type):
pass
@stacked
@with_handling
def visit_ByteType(self, type):
pass
@stacked
@with_handling
def visit_IntegerType(self, type):
pass
@stacked
@with_handling
def visit_FloatType(self, type):
pass
@stacked
@with_handling
def visit_LongType(self, type):
pass
@stacked
@with_handling
def visit_TimestampType(self, type):
pass
@stacked
@with_handling
def visit_ManyType(self, many):
many.subtype.accept(self)
@stacked
@with_handling
def visit_AmountType(self, many):
many.subtype.accept(self)
@stacked
@with_handling
def visit_TupleType(self, tuple):
for type in tuple.types:
type.accept(self)
@stacked
@with_handling
def visit_VariableExp(self, var):
# on first use, a variable is self-instantiating
try:
prev = self.env[var.name]
except KeyError:
# print "AUTO-DECL", var.name, "Stack=", self.stack_as_string(), "Env=", str(self.env)
self.env[var.name] = var
var.identifier.accept(self)
var.type.accept(self)
@stacked
@with_handling
def visit_PropertyExp(self, prop):
prop.obj.accept(self)
prop.identifier.accept(self)
prop.type.accept(self)
@stacked
@with_handling
def visit_UnaryExp(self, exp):
exp.operand.accept(self)
@stacked
@with_handling
def visit_BinaryExp(self, exp):
exp.left.accept(self)
exp.right.accept(self)
@stacked
@with_handling
def visit_NumericBinaryExp(self, exp):
exp.left.accept(self)
exp.right.accept(self)
@stacked
@with_handling
def visit_FunctionCallExp(self, exp):
exp.function.accept(self)
exp.type.accept(self)
for arg in exp.arguments:
arg.accept(self)
@stacked
@with_handling
def visit_FunctionExp(self, exp):
pass
@stacked
@with_handling
def visit_ObjectExp(self, exp):
exp._type.accept(self)
@stacked
@with_handling
def visit_ObjectType(self, obj):
pass
@stacked
@with_handling
def visit_MethodCallExp(self, exp):
exp.object.accept(self)
exp.type.accept(self)
for arg in exp.arguments:
arg.accept(self)
@stacked
@with_handling
def visit_AnythingExp(self, exp):
pass
@stacked
@with_handling
def visit_MatchExp(self, exp):
exp.operator.accept(self)
if exp.operand != None:
exp.operand.accept(self)
@with_handling
def visit_Comparator(self, comp):
pass
class TestEnvironment(unittest.TestCase):
def setUp(self):
self.env = Environment()
def test_empty_environment(self):
self.assertDictEqual(self.env.envs[0], {})
def test_set_get_in_current_environment(self):
self.env['test'] = 'test'
self.assertEqual(self.env['test'], 'test')
def test_set_get_in_previous_environment(self):
self.env['test'] = 'test1'
self.env.extend()
self.env['test'] = 'test2'
self.assertEqual(self.env['test'], 'test2')
def test_set_get_in_reduced_environment(self):
self.env['test'] = 'test1'
self.env.extend()
self.env['test'] = 'test2'
self.env.reduce()
self.assertEqual(self.env['test'], 'test1')
def create_environment(self):
self.env['abc'] = 'abc1'
self.env['def'] = 'def1'
self.env.extend()
self.env['abc'] = 'abc2'
self.env['def'] = 'def2'
def test_multiple_items_in_environment(self):
self.create_environment()
self.assertEqual(self.env['abc'], 'abc2')
def test_unknown_item_in_environment(self):
self.create_environment()
def bad():
self.env['unknown']
self.assertRaises(KeyError, bad)
def test_reduce_too_much(self):
self.env.reduce()
def bad():
self.env.reduce()
self.assertRaises(RuntimeError, bad)
def test_stringification_of_environment(self):
self.create_environment()
self.assertEqual(
str(self.env),
"abc : abc2\ndef : def2\n abc : abc1\n def : def1\n")
def test_contain_in_environment(self):
self.create_environment()
self.assertTrue("abc" in self.env)
self.assertFalse("xyz" in self.env)
def __init__(self, warn_unhandled=False, visit_internal_types=True): self.prefix = None self._stack = [] self.env = Environment() self.warn_unhandled = warn_unhandled self.visit_internal_types = visit_internal_types
class SemanticVisitor(SemanticVisitorBase):
def __init__(self, warn_unhandled=False, visit_internal_types=True):
self.prefix = None
self._stack = []
self.env = Environment()
self.warn_unhandled = warn_unhandled
self.visit_internal_types = visit_internal_types
def get_stack(self): return self._stack
stack = property(get_stack)
def stack_as_string(self):
return " > ".join([str(obj) for obj in self._stack])
# HANDLERS IMPLEMENTING VISITOR and adding a handle call
@stacked
@with_handling
def visit_Identifier(self, id): pass
@stacked
@with_handling
def visit_Model(self, model):
for module in model.modules.values():
module.accept(self)
@stacked
@with_handling
def visit_Domain(self, domain):
self.env[domain.name] = domain
@stacked
@with_handling
def visit_Scope(self, scope): pass
@stacked
@with_handling
def visit_Module(self, module):
self.env.extend()
for constant in module.constants:
constant.accept(self)
for name, function in module.externals.items():
self.env[name] = function
for domain in module.domains:
domain.accept(self)
for extension in domain.extensions:
extension.accept(self)
for function in module.functions:
# we only want to handle non-anonymous function declarations here
# because anonymous declarations are declared in their specific scope
# TODO: do this in a cleaner way (AnonFunctionDecl?)
if function.name[0:9] != "anonymous":
function.accept(self)
for execution in module.executions:
execution.accept(self)
self.env.reduce()
@stacked
@with_handling
def visit_Constant(self, constant):
self.env[constant.name] = constant
@stacked
@with_handling
def visit_Extension(self, extension):
extension.domain.accept(self)
extension.extension.accept(self)
@stacked
@with_handling
def visit_Every(self, execution):
execution.interval.accept(self)
execution.scope.accept(self)
execution.executed.accept(self)
@stacked
@with_handling
def visit_When(self, execution):
execution.scope.accept(self)
execution.event.accept(self)
execution.executed.accept(self)
@stacked
@with_handling
def visit_FunctionDecl(self, function):
self.env[function.name] = function
self.env.extend()
function.type.accept(self)
for param in function.parameters:
param.accept(self)
function.body.accept(self)
self.env.reduce()
@stacked
@with_handling
def visit_Parameter(self, parameter):
self.env[parameter.name] = parameter
parameter.type.accept(self)
# STATEMENTS
@stacked
@with_handling
def visit_BlockStmt(self, block):
self.env.extend()
for statement in block.statements:
statement.accept(self)
self.env.reduce()
@stacked
@with_handling
def visit_AssignStmt(self, stmt):
stmt.value.accept(self)
stmt.variable.accept(self)
@stacked
@with_handling
def visit_AddStmt(self, stmt):
stmt.variable.accept(self)
stmt.value.accept(self)
@stacked
@with_handling
def visit_SubStmt(self, stmt):
stmt.variable.accept(self)
stmt.value.accept(self)
@stacked
@with_handling
def visit_IncStmt(self, stmt):
stmt.variable.accept(self)
@stacked
@with_handling
def visit_DecStmt(self, stmt):
stmt.variable.accept(self)
@stacked
@with_handling
def visit_IfStmt(self, stmt):
stmt.condition.accept(self)
stmt.true.accept(self)
if stmt.false != None:
stmt.false.accept(self)
@stacked
@with_handling
def visit_CaseStmt(self, stmt):
stmt.expression.accept(self)
for case, consequence in zip(stmt.cases, stmt.consequences):
self.env.extend()
case.accept(self)
consequence.accept(self)
self.env.reduce()
@stacked
@with_handling
def visit_ReturnStmt(self, stmt):
if stmt.expression != None:
stmt.expression.accept(self)
# EXPRESSIONS
@stacked
@with_handling
def visit_BooleanLiteralExp(self, exp): pass
@stacked
@with_handling
def visit_IntegerLiteralExp(self, exp): pass
@stacked
@with_handling
def visit_FloatLiteralExp(self, exp): pass
@stacked
@with_handling
def visit_AtomLiteralExp(self, atom): pass
@stacked
@with_handling
def visit_UnknownType(self, lst): pass
def visit_AnyType(self, type): pass
def visit_MixedType(self, type): pass
def visit_AtomType(self, type): pass
@stacked
@with_handling
def visit_ListLiteralExp(self, lst):
if self.visit_internal_types: lst.type.accept(self)
for exp in lst.expressions:
exp.accept(self)
@stacked
@with_handling
def visit_ObjectLiteralExp(self, obj):
for prop in obj.properties:
prop.accept(self)
@stacked
@with_handling
def visit_Property(self, prop):
if prop.type != None:
prop.type.accept(self)
prop.value.accept(self)
# types
@stacked
@with_handling
def visit_VoidType(self, type): pass
@stacked
@with_handling
def visit_BooleanType(self, type): pass
@stacked
@with_handling
def visit_ByteType(self, type): pass
@stacked
@with_handling
def visit_IntegerType(self, type): pass
@stacked
@with_handling
def visit_FloatType(self, type): pass
@stacked
@with_handling
def visit_LongType(self, type): pass
@stacked
@with_handling
def visit_TimestampType(self, type): pass
@stacked
@with_handling
def visit_ManyType(self, many):
many.subtype.accept(self)
@stacked
@with_handling
def visit_AmountType(self, many):
many.subtype.accept(self)
@stacked
@with_handling
def visit_TupleType(self, tuple):
for type in tuple.types:
type.accept(self)
@stacked
@with_handling
def visit_VariableExp(self, var):
# on first use, a variable is self-instantiating
try:
prev = self.env[var.name]
except KeyError:
# print "AUTO-DECL", var.name, "Stack=", self.stack_as_string(), "Env=", str(self.env)
self.env[var.name] = var
var.identifier.accept(self)
var.type.accept(self)
@stacked
@with_handling
def visit_PropertyExp(self, prop):
prop.obj.accept(self)
prop.identifier.accept(self)
prop.type.accept(self)
@stacked
@with_handling
def visit_UnaryExp(self, exp):
exp.operand.accept(self)
@stacked
@with_handling
def visit_BinaryExp(self, exp):
exp.left.accept(self)
exp.right.accept(self)
@stacked
@with_handling
def visit_NumericBinaryExp(self, exp):
exp.left.accept(self)
exp.right.accept(self)
@stacked
@with_handling
def visit_FunctionCallExp(self, exp):
exp.function.accept(self)
exp.type.accept(self)
for arg in exp.arguments:
arg.accept(self)
@stacked
@with_handling
def visit_FunctionExp(self, exp): pass
@stacked
@with_handling
def visit_ObjectExp(self, exp):
exp._type.accept(self)
@stacked
@with_handling
def visit_ObjectType(self, obj): pass
@stacked
@with_handling
def visit_MethodCallExp(self, exp):
exp.object.accept(self)
exp.type.accept(self)
for arg in exp.arguments:
arg.accept(self)
@stacked
@with_handling
def visit_AnythingExp(self, exp): pass
@stacked
@with_handling
def visit_MatchExp(self, exp):
exp.operator.accept(self)
if exp.operand != None:
exp.operand.accept(self)
@with_handling
def visit_Comparator(self, comp): pass
class ec2(object):
'''Methods to use aws ec2 instances'''
# TODO : setting region doesn't actually do anything yet
def __init__(self, aws_access_key=None, aws_secret_key=None, region=None):
'''Default constructor.
Inputs:
aws_access_key = access key provided by aws
aws_secret_key = secret key associated with access key'''
self.env = Environment()
if aws_access_key is None:
self.__access_key = self.env.get("ACCESS_KEY")
else:
self.__access_key = aws_access_key
if aws_secret_key is None:
self.__secret_key = self.env.get("SECRET_KEY")
else:
self.__secret_key = aws_secret_key
self.__region = region
self.__conn = EC2Connection(aws_access_key_id=self.__access_key,
aws_secret_access_key=self.__secret_key)
self.__runningInstances = {}
def getAvailImages(self):
'''Get all available images that an ec2 instance can be started up with.'''
return self.__conn.get_all_images()
def getRunningInstances(self):
return self.__runningInstances
# TODO : need some validation here on inputs
def startInstance(self, name, imageName, instanceType, keyPairName=None):
'''Start up a new ec2 instance.
Inputs:
name = name to associate with instance
imageName = name of image from VALID_IMGS
instanceType = type of instance to start up, must be in the
list for the given VALID_IMGS
keyPairName = key pair to associate with this image
Returns:
dns to server if server successfully starts up'''
if instanceType not in VALID_IMGS[imageName]["supported_instances"]:
raise Exception("'" + instanceType +
"' is not a valid type for '" +
VALID_IMGS[imageName]["ami-76f0061f"] + "'")
if keyPairName is None:
keyPairName = self.env.get("KEY_PAIR")
image = self.__conn.get_image(VALID_IMGS[imageName]["imageid"])
reservation = image.run(instance_type=instanceType,
key_name=keyPairName) #,
#security_groups=["vertex"])
instance = reservation.instances[0]
instance.update()
while instance.state != u'running':
instance.update()
self.__runningInstances[name] = instance
return instance.dns_name
# TODO : might want to have a full list of installable software somewhere
def prepareAndRunInstance(self,
dnsName,
localPipeDir,
scriptName,
softwareList=[],
pipeArgs=None,
keyPairName=None):
'''Set up instance software.
Inputs:
dnsName = dns name of server
localPipeDir = absolute path to directory containing local
pipeline and associated files
scriptName = name of pipeline to run
softwareList = a list of software to install, must be accessible
by yum on an ec2 instance. Best tested by running an ec2
instance and trying yum search or yum list all
pipeArgs = string to pass to pipeline as arguments
keyPairName = name of key used to start instance
Returns:
True if instance is set up and run properly'''
if keyPairName is None:
keyPairName = self.env.get("KEY_PAIR")
keyPairAbsPath = os.path.join(os.getenv("HOME"),
".ssh/%s.pem" % keyPairName)
with settings(host_string="ec2-user@%s" % dnsName,
key_filename=keyPairAbsPath,
warn_only=True):
if not self._port_open(dnsName, 22):
print(
"FATAL: counld not connect to ec2 instance. Try increasing number of retries?"
)
return False
# TODO : might want to have the dist package somewhere online to just download
sudo("yum install -y git")
# TODO : would probably be better to have an image already made with this stuff
sudo("yum install -y python-boto.noarch")
# TODO : need to install fabric, can't figure out how to get fabric on instance, for now...
sudo("git clone git://github.com/kand/aws_pipeline.git")
sudo("chmod +x aws_pipeline/vertex_pipeline")
for s in softwareList:
sudo("yum install -y %s" % s)
put(localPipeDir, '/home/ec2-user/aws_pipeline', use_sudo=True)
if pipeArgs is not None:
sudo(
"aws_pipeline/vertex_pipeline run aws_pipeline/%s/%s.py %s &"
% (scriptName, scriptName, pipeArgs))
else:
sudo("aws_pipeline/vertex_pipeline run aws_pipeline/%s/%s.py &"
% (scriptName, scriptName))
return True
def stopInstance(self, name):
'''Stop an instance. This does not terminate an instance, however, as boto
doesn't seem to have a way to do this...
Inputs:
name = name associated with instance to stop, set when using
startInstance()
Returns:
True if instance was successfully stopped.'''
if name not in self.__runningInstances.keys():
print("Name not found in dict of running instances")
return False
instance = self.__runningInstances[name]
instance.stop()
instance.update()
while instance.state != u'stopped':
print(instance.state)
instance.update()
self.__runningInstances.pop(name)
return True
# this function errors out in boto
#def close(self):a.close()
# '''Close ec2 connection'''
#
# if not self.__checkConn(): return
# self.__conn.close()'''
# pass
def _port_open(self, host, port, retries=None):
'''Test if a port is open on host.
Inputs:
host = host to attempt to connect to
port = port to attempt to connect on
retries = number of retries to attempt, with a 1 second pause between
Returns:
True if connected, False if failed'''
if retries is None:
retries = int(self.env.get("MAX_SSH_RETRIES"))
print("attempting to connect")
for i in range(0, retries):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((host, port))
sock.close()
print("connected")
return True
except Exception:
print("."),
time.sleep(1)
continue
print("connected")
return False
def setUp(self):
self.env = Environment()
def __init__(self):
from util.environment import Environment
self._configuration_directory = Environment.get_configuration_path()
def setUp(self): self.env = Environment()
class TestEnvironment(unittest.TestCase):
def setUp(self):
self.env = Environment()
def test_empty_environment(self):
self.assertDictEqual(self.env.envs[0], {})
def test_set_get_in_current_environment(self):
self.env['test'] = 'test'
self.assertEqual(self.env['test'], 'test')
def test_set_get_in_previous_environment(self):
self.env['test'] = 'test1'
self.env.extend()
self.env['test'] = 'test2'
self.assertEqual(self.env['test'], 'test2')
def test_set_get_in_reduced_environment(self):
self.env['test'] = 'test1'
self.env.extend()
self.env['test'] = 'test2'
self.env.reduce()
self.assertEqual(self.env['test'], 'test1')
def create_environment(self):
self.env['abc'] = 'abc1'
self.env['def'] = 'def1'
self.env.extend()
self.env['abc'] = 'abc2'
self.env['def'] = 'def2'
def test_multiple_items_in_environment(self):
self.create_environment()
self.assertEqual(self.env['abc'], 'abc2')
def test_unknown_item_in_environment(self):
self.create_environment()
def bad(): self.env['unknown']
self.assertRaises(KeyError, bad)
def test_reduce_too_much(self):
self.env.reduce()
def bad(): self.env.reduce()
self.assertRaises(RuntimeError, bad)
def test_stringification_of_environment(self):
self.create_environment()
self.assertEqual(str(self.env),
"abc : abc2\ndef : def2\n abc : abc1\n def : def1\n" )
def test_contain_in_environment(self):
self.create_environment()
self.assertTrue("abc" in self.env)
self.assertFalse("xyz" in self.env)
def __init__(self, warn_unhandled=False, visit_internal_types=True):
self.prefix = None
self._stack = []
self.env = Environment()
self.warn_unhandled = warn_unhandled
self.visit_internal_types = visit_internal_types