def open_from_db(self, db_connection, vtType, id=None, lock=False, global_props=None): all_objects = {} if global_props is None: global_props = {} if id is not None: global_props['id'] = id # print global_props res_objects = self['sql'][vtType].get_sql_columns( db_connection, global_props, lock) if len(res_objects) > 1: raise VistrailsDBException("More than object of type '%s' and " "id '%s' exist in the database" % \ (vtType, id)) elif len(res_objects) <= 0: raise VistrailsDBException("No objects of type '%s' and " "id '%s' exist in the database" % \ (vtType, id)) all_objects.update(res_objects) res = res_objects.values()[0] global_props = {'entity_id': res.db_id, 'entity_type': res.vtType} for dao_type, dao in self['sql'].iteritems(): if (dao_type == DBVistrail.vtType or dao_type == DBWorkflow.vtType or dao_type == DBLog.vtType or dao_type == DBRegistry.vtType): continue current_objs = dao.get_sql_columns(db_connection, global_props, lock) all_objects.update(current_objs) if dao_type == DBGroup.vtType: for key, obj in current_objs.iteritems(): new_props = { 'parent_id': key[1], 'entity_id': global_props['entity_id'], 'entity_type': global_props['entity_type'] } res_obj = self.open_from_db(db_connection, DBWorkflow.vtType, None, lock, new_props) res_dict = {} res_dict[(res_obj.vtType, res_obj.db_id)] = res_obj all_objects.update(res_dict) for key, obj in all_objects.iteritems(): if key[0] == vtType and key[1] == id: continue self['sql'][obj.vtType].from_sql_fast(obj, all_objects) for obj in all_objects.itervalues(): obj.is_dirty = False obj.is_new = False return res
def open_from_db(self, db_connection, vtType, id, lock=False): all_objects = {} global_props = {'id': id} # print global_props res_objects = self['sql'][vtType].get_sql_columns(db_connection, global_props, lock) if len(res_objects) > 1: raise VistrailsDBException("More than object of type '%s' and " "id '%s' exist in the database" % \ (vtType, id)) elif len(res_objects) <= 0: raise VistrailsDBException("No objects of type '%s' and " "id '%s' exist in the database" % \ (vtType, id)) all_objects.update(res_objects) res = res_objects.values()[0] del global_props['id'] for dao in self['sql'].itervalues(): if (dao == self['sql'][DBVistrail.vtType] or # dao == self['sql'][DBWorkflow.vtType] or dao == self['sql'][DBLog.vtType] or dao == self['sql'][DBAbstraction.vtType]): continue current_objs = dao.get_sql_columns(db_connection, global_props, lock) if dao == self['sql'][DBWorkflow.vtType]: for key, obj in current_objs.iteritems(): if key[0] == vtType and key[1] == id: continue elif key[0] == DBWorkflow.vtType: res_obj = self.open_from_db(db_connection, key[0], key[1], lock) res_dict = {} res_dict[(res_obj.db_id, res_obj.vtType)] = res_obj all_objects.update(res_dict) else: all_objects.update(current_objs) for key, obj in all_objects.iteritems(): if key[0] == vtType and key[1] == id: continue self['sql'][obj.vtType].from_sql_fast(obj, all_objects) for obj in all_objects.itervalues(): obj.is_dirty = False obj.is_new = False return res
def parse_xml_file(filename): try: return xml.dom.minidom.parse(filename) except xml.parsers.expat.ExpatError, e: msg = 'XML parse error at line %s, col %s: %s' % \ (e.lineno, e.offset, e.code) raise VistrailsDBException(msg)
def convert_filename_to_url(cls, filename): """ Converts a local filename to a file:// URL. All file:// URLs are absolute, so abspath() will be used on the argument. """ exts = ["vt", "xml"] q_mark = False query_str_idx = None for match in re.finditer("\.(%s)(\??)" % "|".join(exts), filename): if match.group(2): if q_mark: raise VistrailsDBException('Ambiguous URI with ' 'multiple question ' 'marks: "%s"' % filename) else: q_mark = True query_str_idx = match.end() if q_mark: args_str = filename[query_str_idx-1:] filename = filename[:query_str_idx-1] else: args_str = "" return 'file://%s%s' % (pathname2url(cls.real_filename(filename)), urllib.quote(args_str, safe='/?=&'))
def getVersionDAO(version=None): if version is None: version = currentVersion persistence_dir = 'vistrails.db.versions.' + get_version_name(version) + \ '.persistence' try: persistence = __import__(persistence_dir, {}, {}, ['']) except ImportError as e: if str(e).startswith('No module named v'): # assume version is not available msg = "Cannot find DAO for version '%s'" % version raise VistrailsDBException(msg) # assume other error import traceback raise VistrailsDBException(debug.format_exc()) return persistence.DAOList()
def executeSQLGroup(self, db, dbCommandList, isFetch): """ Executes a command consisting of multiple SELECT statements It returns a list of results from the SELECT statements """ data = [] # break up into bundles BUNDLE_SIZE = 10000 num_commands = len(dbCommandList) n = 0 while n<num_commands: dbCommands = dbCommandList[n:(n+BUNDLE_SIZE)] commandString = '' for prepared, values in dbCommands: command = prepared % \ tuple(db.escape(v, get_db_lib().converters.conversions) for v in values) commandString += command cur = db.cursor() try: result = cur.execute(commandString) while True: r = cur.fetchall() if isFetch else cur.lastrowid data.append(r) next = cur.nextset() if not next: break except Exception, e: raise VistrailsDBException('Command failed: %s -- """ %s """' % (e, commandString)) finally:
def unserialize(self, str, obj_type): try: root = ElementTree.fromstring(str) return self.read_xml_object(obj_type, root) except SyntaxError, e: msg = "Invalid VisTrails serialized object %s" % str raise VistrailsDBException(msg) return None
def get_autosave_dir(): dot_vistrails = vistrails.core.system.current_dot_vistrails() auto_save_dir = os.path.join(dot_vistrails, "autosave") if not os.path.exists(auto_save_dir): # !!! we assume dot_vistrails exists !!! os.mkdir(auto_save_dir) if not os.path.isdir(auto_save_dir): raise VistrailsDBException('Auto-save path "%s" is not a ' 'directory' % auto_save_dir) return auto_save_dir
def translateVistrail(vistrail): id_remap = {} for action in vistrail.db_get_actions(): # don't need to change key idx since none of that changes new_action_idx = {} for annotation in action.db_get_annotations(): annotation.db_id = vistrail.idScope.getNewId(DBAnnotation.vtType) new_action_idx[annotation.db_id] = annotation action.db_annotations_id_index = new_action_idx for operation in action.db_get_operations(): # never have annotations as parent objs so # don't have to worry about those ids if operation.db_what == DBAnnotation.vtType: if operation.vtType == 'add': new_id = vistrail.idScope.getNewId(DBAnnotation.vtType) old_id = operation.db_objectId operation.db_objectId = new_id operation.db_data.db_id = new_id id_remap[old_id] = new_id elif operation.vtType == 'change': changed_id = operation.db_oldObjId if id_remap.has_key(changed_id): operation.db_oldObjId = id_remap[changed_id] else: raise VistrailsDBException('cannot translate') new_id = vistrail.idScope.getNewId(DBAnnotation.vtType) old_id = operation.db_newObjId operation.db_newObjId = new_id operation.db_data.db_id = new_id id_remap[old_id] = new_id elif operation.vtType == 'delete': old_id = operation.db_objectId if id_remap.has_key(old_id): operation.db_objectId = id_remap[old_id] else: raise VistrailsDBException('cannot translate') vistrail.db_version = '0.8.1' return vistrail
def executeSQL(self, db, cmd_tuple, isFetch): dbCommand, values = cmd_tuple # print 'db: %s' % dbCommand # print 'values:', values data = None cursor = db.cursor() try: cursor.execute(dbCommand, values) if isFetch: data = cursor.fetchall() else: data = cursor.lastrowid except Exception, e: raise VistrailsDBException('Command "%s" with values "%s" ' 'failed: %s' % (dbCommand, values, e))
def delete_from_db(self, db_connection, type, obj_id): if type not in root_set: raise VistrailsDBException("Cannot delete entity of type '%s'" \ % type) id_str = str(obj_id) for (dao_type, dao) in self['sql'].iteritems(): if dao_type not in root_set: db_cmd = \ self['sql'][type].createSQLDelete(dao.table, {'entity_type': type, 'entity_id': id_str}) self['sql'][type].executeSQL(db_connection, db_cmd, False) db_cmd = self['sql'][type].createSQLDelete(self['sql'][type].table, {'id': id_str}) self['sql'][type].executeSQL(db_connection, db_cmd, False)
def unserialize(self, str, obj_type): def set_dirty(obj): for child, _, _ in obj.db_children(): if child.vtType == DBGroup.vtType: if child.db_workflow: set_dirty(child.db_workflow) child.is_dirty = True child.is_new = True try: root = ElementTree.fromstring(str) obj = self.read_xml_object(obj_type, root) set_dirty(obj) return obj except SyntaxError, e: msg = "Invalid VisTrails serialized object %s" % str raise VistrailsDBException(msg) return None
def from_url(url): if not url.startswith('untitled:'): raise VistrailsDBException("URL does not start with untitled:") rest = url[9:] my_uuid = None if len(rest) >= 32: try: my_uuid = uuid.UUID(rest[:32]) rest = rest[32:] except ValueError: pass if not rest: kwargs = dict() elif rest[0] == '?': kwargs = BaseLocator.parse_args(rest[1:]) else: raise ValueError return UntitledLocator(my_uuid, **kwargs)
def open_many_from_db(self, db_connection, vtType, ids, lock=False): """ Loads multiple objects. They need to be loaded as one single multiple select statement command for performance reasons. """ log_dao = self['sql'][vtType] # loop through ids and build SELECT statements selects = [ log_dao.get_sql_select(db_connection, {'id': id}, lock) for id in ids ] # Execute all SELECT statements for main objects results = log_dao.executeSQLGroup(db_connection, selects, True) # list of final objects objects = [] # list of selects selects = [] # list of children id:all_objects_dict all_objects_dict = {} # process each result and extract child SELECTS # daoList should contain (id, dao_type, dao, result) values daoList = [] # selects should contain dbCommand values selects = [] global_props = {} for id, data in zip(ids, results): res_objects = log_dao.process_sql_columns(data, global_props) if len(res_objects) > 1: raise VistrailsDBException("More than object of type '%s' and " "id '%s' exist in the database" % \ (vtType, id)) elif len(res_objects) <= 0: raise VistrailsDBException("No objects of type '%s' and " "id '%s' exist in the database" % \ (vtType, id)) all_objects = {} all_objects_dict[id] = all_objects all_objects.update(res_objects) objects.append(res_objects.values()[0]) # collect all commands so that they can be executed together # generate SELECT statements for children for dao_type, dao in self['sql'].iteritems(): if (dao_type == DBVistrail.vtType or dao_type == DBWorkflow.vtType or dao_type == DBLog.vtType or dao_type == DBRegistry.vtType): continue daoList.append([id, dao_type, dao, None]) dbCommand = dao.get_sql_select(db_connection, global_props, lock) selects.append(dbCommand) # Execute all child select statements results = self['sql'][vtType].executeSQLGroup(db_connection, selects, True) for i in xrange(len(daoList)): daoList[i][3] = results[i] # process results for id, dao_type, dao, data in daoList: all_objects = all_objects_dict[id] current_objs = dao.process_sql_columns(data, global_props) all_objects.update(current_objs) if dao_type == DBGroup.vtType: for key, obj in current_objs.iteritems(): new_props = { 'parent_id': key[1], 'entity_id': global_props['entity_id'], 'entity_type': global_props['entity_type'] } res_obj = self.open_from_db(db_connection, DBWorkflow.vtType, None, lock, new_props) res_dict = {} res_dict[(res_obj.vtType, res_obj.db_id)] = res_obj all_objects.update(res_dict) for id, all_objects in all_objects_dict.iteritems(): for key, obj in all_objects.iteritems(): if key[0] == vtType and key[1] == id: continue self['sql'][obj.vtType].from_sql_fast(obj, all_objects) for id, dao_type, dao, data in daoList: all_objects = all_objects_dict[id] for obj in all_objects.itervalues(): obj.is_dirty = False obj.is_new = False return objects
def open_from_db(self, db_connection, vtType, id=None, lock=False, global_props=None): all_objects = {} if global_props is None: global_props = {} if id is not None: global_props['id'] = id # print global_props res_objects = self['sql'][vtType].get_sql_columns( db_connection, global_props, lock) if len(res_objects) > 1: raise VistrailsDBException("More than object of type '%s' and " "id '%s' exist in the database" % \ (vtType, id)) elif len(res_objects) <= 0: raise VistrailsDBException("No objects of type '%s' and " "id '%s' exist in the database" % \ (vtType, id)) all_objects.update(res_objects) res = res_objects.values()[0] global_props = {'entity_id': res.db_id, 'entity_type': res.vtType} # collect all commands so that they can be executed together # daoList should contain (dao_type, dao, dbCommand) values daoList = [] # dbCommandList should contain dbCommand values dbCommandList = [] # generate SELECT statements for dao_type, dao in self['sql'].iteritems(): if (dao_type == DBVistrail.vtType or dao_type == DBWorkflow.vtType or dao_type == DBLog.vtType or dao_type == DBRegistry.vtType): continue daoList.append([dao_type, dao, None]) dbCommand = dao.get_sql_select(db_connection, global_props, lock) dbCommandList.append(dbCommand) # Exacute all select statements results = self['sql'][vtType].executeSQLGroup(db_connection, dbCommandList, True) # add result to correct dao for i in xrange(len(daoList)): daoList[i][2] = results[i] # process results for dao_type, dao, data in daoList: current_objs = dao.process_sql_columns(data, global_props) all_objects.update(current_objs) if dao_type == DBGroup.vtType: for key, obj in current_objs.iteritems(): new_props = { 'parent_id': key[1], 'entity_id': global_props['entity_id'], 'entity_type': global_props['entity_type'] } res_obj = self.open_from_db(db_connection, DBWorkflow.vtType, None, lock, new_props) res_dict = {} res_dict[(res_obj.vtType, res_obj.db_id)] = res_obj all_objects.update(res_dict) for key, obj in all_objects.iteritems(): if key[0] == vtType and key[1] == id: continue self['sql'][obj.vtType].from_sql_fast(obj, all_objects) for obj in all_objects.itervalues(): obj.is_dirty = False obj.is_new = False return res
def runWorkflowQuery(config, vistrail=None, version=None, fromTime=None, toTime=None, user=None, offset=0, limit=100, modules=[], thumbs=None): # returns list of workflows: # (vistrail name, vistrail id, id, name, date, user, thumb) result = [] db = open_db_connection(config) select_part = \ """SELECT DISTINCT v.name, v.id, w.parent_id, a1.value, action.date, action.user""" from_part = \ """FROM workflow w""" # "tag name" exist in workflow table but may have been changed # so we use value from the vistrail __tag__ annotation where_part = \ """WHERE w.entity_type='workflow'""" limit_part = 'LIMIT %s, %s' % (int(offset), int(limit)) if vistrail: try: where_part += " AND v.id=%s" % int(vistrail) except ValueError: where_part += " AND v.name=%s" % \ db.escape(vistrail, get_db_lib().converters.conversions) if version: try: where_part += " AND w.parent_id=%s" % int(version) except ValueError: where_part += " AND a1.value=%s" % \ db.escape(version, get_db_lib().converters.conversions) if fromTime: where_part += " AND w.last_modified>%s" % \ db.escape(fromTime, get_db_lib().converters.conversions) if toTime: where_part += " AND w.last_modified<%s" % \ db.escape(toTime, get_db_lib().converters.conversions) if user: where_part += " AND action.user=%s" % \ db.escape(user, get_db_lib().converters.conversions) next_port = 1 old_alias = None for i, module, connected in zip(range(1,len(modules)+1), *zip(*modules)): module = module.lower() alias = "m%s"%i from_part += \ """ JOIN module {0} ON ({0}.parent_id=w.id AND {0}.entity_type=w.entity_type AND {0}.name={1}) """.format(alias, db.escape(module, get_db_lib().converters.conversions)) if connected: p1_alias, p2_alias=("port%s"%next_port), ("port%s"%(next_port+1)) next_port += 2 from_part += \ """ JOIN port {0} ON ({0}.entity_id=w.id AND {0}.entity_type=w.entity_type AND {0}.moduleId={1}.id AND {0}.type='source')""".format( p1_alias, old_alias) from_part += \ """ JOIN port {0} ON ({0}.entity_id=w.id AND {0}.entity_type=w.entity_type AND {0}.moduleId={1}.id AND {0}.type='destination' AND {0}.parent_id = {2}.parent_id)""".format( p2_alias, alias, p1_alias) old_alias = alias from_part += \ """ JOIN vistrail v ON w.vistrail_id = v.id JOIN action ON action.entity_id=w.vistrail_id AND action.id=w.parent_id LEFT JOIN action_annotation a1 ON a1.entity_id=w.vistrail_id AND a1.action_id=w.parent_id AND (a1.akey='__tag__' OR a1.akey IS NULL)""" if thumbs: select_part += ', t.image_bytes' from_part += """ LEFT JOIN action_annotation a2 ON (a2.entity_id=w.vistrail_id AND a2.action_id=w.parent_id AND (a2.akey='__thumb__' OR a2.akey IS NULL)) LEFT JOIN thumbnail t ON a2.value=t.file_name""" else: select_part += ', NULL' command = ' '.join([select_part, from_part, where_part, limit_part]) + ';' #print command try: c = db.cursor() c.execute(command) rows = c.fetchall() result = rows c.close() except get_db_lib().Error, e: msg = "Couldn't perform query on db (%d : %s)" % \ (e.args[0], e.args[1]) raise VistrailsDBException(msg)
def runLogQuery(config, vistrail=None, version=None, fromTime=None, toTime=None, user=None, completed=None, offset=0, limit=100, modules=[], thumbs=None): # returns list of workflow executions: # (vistrail name, vistrail id, log id, workflow id, workflow name, # execution id, start time, end time, user, completed, thumb) result = [] db = open_db_connection(config) select_part = \ """SELECT DISTINCT v.name, v.id, w.entity_id, w.parent_version, a1.value, w.id, w.ts_start, w.ts_end, w.user, w.completed""" from_part = \ """FROM workflow_exec w JOIN log_tbl l ON (l.id = w.entity_id) JOIN vistrail v ON (l.vistrail_id = v.id) LEFT JOIN action_annotation a1 ON (a1.entity_id=v.id AND a1.action_id=w.parent_version)""" where_part = \ """WHERE w.parent_type='vistrail' AND w.entity_type='log' AND (a1.akey='__tag__' OR a1.akey IS NULL)""" limit_part = 'LIMIT %s, %s' % (int(offset), int(limit)) if vistrail: try: where_part += " AND v.id=%s" % int(vistrail) except ValueError: where_part += " AND v.name=%s" % \ db.escape(vistrail, get_db_lib().converters.conversions) if version: try: where_part += " AND w.parent_version=%s" % int(version) except ValueError: where_part += " AND a1.value=%s" % \ db.escape(version, get_db_lib().converters.conversions) if fromTime: where_part += " AND w.ts_end>%s" % \ db.escape(fromTime, get_db_lib().converters.conversions) if toTime: where_part += " AND w.ts_start<%s" % \ db.escape(toTime, get_db_lib().converters.conversions) if user: where_part += " AND w.user=%s" % \ db.escape(user, get_db_lib().converters.conversions) completed_dict = {'no':0, 'yes':1, 'ok':1} if completed is not None: try: int(completed) except ValueError: completed = completed_dict.get(str(completed).lower(), -1) where_part += " AND w.completed=%s" % completed if thumbs: select_part += ', t.image_bytes' from_part += """ LEFT JOIN action_annotation a2 ON (a2.entity_id=v.id AND a2.action_id=w.parent_version) LEFT JOIN thumbnail t ON a2.value=t.file_name""" where_part += " AND (a2.akey='__thumb__' OR a2.akey IS NULL)" else: select_part += ', NULL' # TODO nested module executions are not detected for i, module, mCompleted in zip(range(1,len(modules)+1), *zip(*modules)): alias = "m%s"%i from_part += \ """ JOIN module_exec %s ON (%s.parent_id=w.id AND %s.entity_id=w.entity_id AND %s.entity_type=w.entity_type) """.replace('%s', alias) where_part += \ """ AND %s.parent_type='workflow_exec' AND %s.module_name=%s """ % (alias, alias, db.escape(module.lower(), get_db_lib().converters.conversions) ) if mCompleted is not None: mCompleted = completed_dict.get(str(mCompleted).lower(), -1) where_part += """ AND %s.completed=%s""" % (alias, mCompleted) command = ' '.join([select_part, from_part, where_part, limit_part]) + ';' #print command try: c = db.cursor() c.execute(command) rows = c.fetchall() result = rows c.close() except get_db_lib().Error, e: msg = "Couldn't perform query on db (%d : %s)" % \ (e.args[0], e.args[1]) raise VistrailsDBException(msg)
# count all rows when offset = 0 if 0 == offset: select_part = 'SELECT count(0)' command = ' '.join([select_part,from_part,where_part]) +';' #print command try: c = db.cursor() c.execute(command) res = c.fetchall() result= (result, res[0][0]) c.close() except get_db_lib().Error, e: msg = "Couldn't perform query on db (%d : %s)" % \ (e.args[0], e.args[1]) raise VistrailsDBException(msg) close_db_connection(db) return result def runLogQuery(config, vistrail=None, version=None, fromTime=None, toTime=None, user=None, completed=None, offset=0, limit=100, modules=[], thumbs=None): # returns list of workflow executions: # (vistrail name, vistrail id, log id, workflow id, workflow name, # execution id, start time, end time, user, completed, thumb) result = [] db = open_db_connection(config) select_part = \ """SELECT DISTINCT v.name, v.id, w.entity_id, w.parent_version, a1.value, w.id,
def translate_object(obj, method_name, version=None, target_version=None): if version is None: version = obj.version if target_version is None: target_version = currentVersion version_map = { '0.3.0': '0.3.1', '0.3.1': '0.6.0', '0.5.0': '0.6.0', '0.6.0': '0.7.0', '0.7.0': '0.8.0', '0.8.0': '0.8.1', '0.8.1': '0.9.0', '0.9.0': '0.9.1', '0.9.1': '0.9.3', '0.9.2': '0.9.3', '0.9.3': '0.9.4', '0.9.4': '0.9.5', '0.9.5': '1.0.0', '1.0.0': '1.0.1', '1.0.1': '1.0.2', '1.0.2': '1.0.3', '1.0.3': '1.0.4', '1.0.4': '1.0.5', } rev_version_map = { '1.0.5': '1.0.4', '1.0.4': '1.0.3', '1.0.3': '1.0.2', '1.0.2': '1.0.1', '1.0.1': '1.0.0', '1.0.0': '0.9.5', '0.9.5': '0.9.4', '0.9.4': '0.9.3', } def get_translate_module(map, start_version, end_version): translate_dir = 'vistrails.db.versions.' + \ get_version_name(end_version) + '.translate.' + \ get_version_name(start_version) return __import__(translate_dir, {}, {}, ['']) path = [] old_tuple = version.split('.') new_tuple = target_version.split('.') map = version_map for i, j in izip(old_tuple, new_tuple): if i < j: # forward break elif i > j: # reverse map = rev_version_map break # don't get stuck in an infinite loop count = 0 while version != target_version: if count > len(map): break next_version = map[version] try: translate_module = get_translate_module(map, version, next_version) except Exception, e: import traceback raise VistrailsDBException("Cannot translate version: " "error loading translation version %s method '%s': %s" % \ (version, method_name, traceback.format_exc())) if not hasattr(translate_module, method_name): raise VistrailsDBException("Cannot translate version: " "version %s missing method '%s'" % \ (version, method_name)) obj = getattr(translate_module, method_name)(obj) version = next_version count += 1