def validate_eids(eids):
if len(eids) < 1:
print("No experiment ids found, nothing to do.")
sys.exit()
for eid in eids:
if not plbCore.validID(eid):
print("Experiment id '" + eid + "' contains invalid characters.")
sys.exit()
def create(labbook_id, ids):
# verify
if not plbCore.validID(args.labbook):
print("'" + labbook_id + "' contains invalid characters")
sys.exit()
# does module exist already?
modfile = labbook_id + ".py"
modffn = os.path.join(plbLabbookRoot, modfile)
if os.path.isfile(modffn):
printwarn("A module already exists for " + labbook_id + ".")
print("You can delete this labbook by using the delete command.")
sys.exit()
if args.dbfile != "":
if not os.path.isfile(args.dbfile):
print("Can't find database file " + args.dbfile + "...")
sys.exit()
try:
labbook = pyLabbook(
id=labbook_id,
root=plbRoot,
repositoryPath=os.path.join(plbRepositoryRoot_REL, labbook_id),
sheetFormat=args.sheetformat,
databasePath=plbDatabaseRoot_REL,
databaseFile=labbook_id + ".sqlite3",
databaseFormat=args.dbformat,
)
except Exception as e:
print("Error initializing: " + str(e))
sys.exit()
serialized = labbook.exportSerialized()
with open(modffn, 'w') as fh:
fh.write(labbook.exportSerialized())
# now try to import and create structure
try:
labbook = plbCore.import_initialize_labbook(labbook_id, plbRoot)
except Exception as e:
print("error importing: " + str(e))
sys.exit()
try:
labbook.createFileStructure()
except Exception as e:
print("error creating repository: " + str(e))
sys.exit()
if args.dbfile != "":
try:
dstfile = os.path.join(plbRoot, labbook.databasePath,
labbook.databaseFile)
plbCore.copyfile(args.dbfile, dstfile)
except Exception as e:
print("error copying " + args.dbfile + ": " + str(e))
print("Labbook will be created with empty database.")
print("You can copy " + args.dbfile + " manually to: \n" + dstfile)
print("Created " + ansi.magenta + modffn + ansi.clear)
print("Done.")
def delete(protocol_id):
# verify id
if not plbCore.validID(protocol_id):
print("'" + protocol_id + "' contains invalid characters")
sys.exit()
# check for no module
modfile = protocol_id + ".py"
modffn = os.path.join(plbProtocolRoot, modfile)
if not os.path.isfile(modffn):
print("Can't find a module for " + protocol_id + "...")
print("Exiting.")
sys.exit()
# warn user
printwarn("Without a module file for " + protocol_id + " you will " +
ansi.br_red + "NOT" + ansi.clear +
" be able to access any of its " + "data in any labbook.")
printwarn("You will have to recreate this module exactly, to access " +
"those data.")
try:
rsp = prompt("Are you really sure you want to do this", ['y', 'n'],
default='n')
if rsp == 'n':
print("Phew! That was close...")
sys.exit()
except Exception as e:
print("Sorry, I don't understand that...")
sys.exit()
# one more time
try:
rsp = prompt("Seriously", ['y', 'n'], default='n')
if rsp == 'n':
print("Wow! That was really close...")
sys.exit()
except Exception as e:
print("Sorry, I don't understand that...")
sys.exit()
print(ansi.red + "DELETING PROTOCOL MODULE " + ansi.br_red + protocol_id +
ansi.red + "..." + ansi.clear)
try:
plbCore.rmfile(modffn)
except Exception as e:
print("Error deleting " + modffn + ": " + str(e))
sys.exit()
print("Successfully deleted " + modffn + ".")
print("Done")
def remove(labbook_id, ids):
if len(ids) < 1:
print("No protocol ids specified... Nothing to do.")
sys.exit()
# validate ids
if not plbCore.validID(args.labbook):
print("'" + labbook_id + "' contains invalid characters")
sys.exit()
for pid in ids:
if not plbCore.validID(pid):
print("'" + pid + "' contains invalid characters")
sys.exit()
# warn user
printwarn("About to remove repository paths for:")
print("\n".join([
ansi.magenta + labbook_id + ansi.cyan + '@' + ansi.magenta + pid +
ansi.clear for pid in ids
]))
print("This will delete ALL associated files and subfolders including +"
"raw data files.")
try:
if prompt("is this what you want", ['y', 'n'], default='n') == 'n':
print("Okay, exiting.")
sys.exit()
except Exception as e:
print("Sorry, I didn't understand that.")
sys.exit()
# confirm
try:
if prompt("Are your really really sure", ['y', 'n'],
default='n') == 'n':
print("Okay, exiting.")
sys.exit()
except Exception as e:
print("Sorry, I didn't understand that.")
sys.exit()
# initialize labbook and import protocols
try:
labbook = plbCore.import_initialize_labbook(labbook_id, plbRoot)
except Exception as e:
print("Error importing " + labbook_id + ": " + str(e))
sys.exit()
print("Removing protocol paths:")
for pid in ids:
try:
protocol = plbCore.import_initialize_protocol(pid, labbook)
except Exception as e:
print(ansi.red + "ERROR" + ansi.clear)
print("couldn't initialize: " + str(e))
continue
print("Deleting path " + ansi.green + protocol.protocolroot() +
ansi.clear + ": ",
end='',
flush=True)
try:
plbCore.rmpath(protocol.protocolroot(), require_empty=False)
except Exception as e:
print(ansi.red + "ERROR" + ansi.clear)
print("Couldn't delete: " + str(e))
continue
print(ansi.green + "OK" + ansi.clear)
print("Done.")
def drop(labbook_id, ids):
if len(ids) < 1:
print("No protocol ids specified... Nothing to do.")
sys.exit()
# validate ids
if not plbCore.validID(args.labbook):
print("'" + labbook_id + "' contains invalid characters")
sys.exit()
for pid in ids:
if not plbCore.validID(pid):
print("'" + pid + "' contains invalid characters")
sys.exit()
# warn user
printwarn("About to drop the following protocol set/sample tables:")
print("\n".join([
ansi.magenta + labbook_id + ansi.cyan + '@' + ansi.magenta + pid +
ansi.clear for pid in ids
]))
print("This will delete all associated database records.")
try:
if prompt("is this what you want", ['y', 'n'], default='n') == 'n':
print("Okay, exiting.")
sys.exit()
except Exception as e:
print("Sorry, I didn't understand that.")
sys.exit()
# confirm
try:
if prompt("Are your really really sure", ['y', 'n'],
default='n') == 'n':
print("Okay, exiting.")
sys.exit()
except Exception as e:
print("Sorry, I didn't understand that.")
sys.exit()
# initialize labbook and import protocols
try:
labbook = plbCore.import_initialize_labbook(labbook_id, plbRoot)
except Exception as e:
print("Error importing " + labbook_id + ": " + str(e))
sys.exit()
print("Dropping protocol tables:")
for pid in ids:
print(ansi.magenta + labbook.id + ansi.cyan + '@' + ansi.magenta +
pid + ansi.clear + ": ",
end='',
flush=True)
try:
protocol = plbCore.import_initialize_protocol(pid, labbook)
except Exception as e:
print(ansi.red + "ERROR" + ansi.clear)
print("couldn't initialize: " + str(e))
continue
protocol.connect()
try:
protocol.dropSetSampleTables()
except Exception as e:
print(ansi.red + "ERROR" + ansi.clear)
print("Couldn't drop tables: " + str(e))
continue
protocol.disconnect()
print(ansi.green + "OK" + ansi.clear)
print("Done.")
def delete(labbook_id, ids):
# verify
if not plbCore.validID(args.labbook):
print("'" + labbook_id + "' contains invalid characters")
sys.exit()
# does module exist?
modfile = labbook_id + ".py"
modffn = os.path.join(plbLabbookRoot, modfile)
if not os.path.isfile(modffn):
print("No module found for " + labbook_id + "... Nothing to do.")
sys.exit()
# warn user
printwarn(ansi.red + "This action will delete all repository files and " +
"databases belonging to " + labbook_id + "." + ansi.clear)
try:
if prompt("Are you really really sure", ['y', 'n'],
default='n') == 'n':
print("Phew! That would have been awful...")
sys.exit()
except Exception as e:
print("Sorry, I don't understand that... Exiting.")
sys.exit()
# confirm
try:
if prompt("Are you sure you're sure", ['y', 'n'], default='n') == 'n':
print("Phew! That would have been awful...")
sys.exit()
except Exception as e:
print("Sorry, I don't understand that... Exiting.")
sys.exit()
# import and initialize and delete...
try:
labbook = plbCore.import_initialize_labbook(labbook_id, plbRoot)
except Exception as e:
print("can't import " + labbook_id + ": " + str(e))
sys.exit()
print("Deleting " + ansi.magenta + labbook.id + ansi.clear +
" repository: ",
end='',
flush=True)
try:
labbook.deleteFileStructure(require_empty=False)
except Exception as e:
print(ansi.red + "ERROR" + ansi.clear)
print(str(e))
sys.exit()
print(ansi.green + "OK" + ansi.clear)
print("Deleting " + ansi.magenta + modffn + ansi.clear + ": ",
end='',
flush=True)
try:
os.remove(modffn)
except Exception as e:
print(ansi.red + "ERROR" + ansi.clear)
print(str(e))
sys.exit()
print(ansi.green + "OK" + ansi.clear)
print("Done.")
print(ansi.red + "ERROR" + ansi.clear)
print("Couldn't delete: " + str(e))
continue
print(ansi.green + "OK" + ansi.clear)
print("Done.")
################################################################################
# Argument Processing
################################################################################
# dict of available commands mapped to their function refs
command_map = {
'create': create,
'delete': delete,
'drop': drop,
'remove': remove,
}
# validate command
if not args.command in command_map.keys():
print("Unrecognized command '" + args.command + "'...")
sys.exit()
# validate labbook
if not plbCore.validID(args.labbook):
print("'" + args.labbook + "' contains invalid characters")
sys.exit()
# call command
command_map[args.command](args.labbook, args.ids)
#
'drop': drop,
}
# validate command
if not args.command in command_map.keys():
print("Unrecognized command '" + args.command + "'...")
sys.exit()
# validate source format
if not '@' in args.source:
print("Source should be formatted as labbook_id@protocol_id.")
sys.exit()
source_parse = args.source.split('@')
if len(source_parse) != 2:
print("Source should be formatted as labbook_id@protocol_id.")
sys.exit()
labbook_id, protocol_id = source_parse[0:2]
if not plbCore.validID(labbook_id):
print("labbook '" + labbook_id + "' has invalid characters.")
sys.exit()
if not plbCore.validID(protocol_id):
print("labbook '" + protocol_id + "' has invalid characters.")
sys.exit()
# check for source labbook and protocol modules
protocol_module = protocol_id + '.py'
labbook_module = labbook_id + '.py'
if not os.path.isfile(os.path.join(plbLabbookRoot, labbook_module)):
print("Can't find labbook module file " + labbook_module + ".")
sys.exit()
if not os.path.isfile(os.path.join(plbProtocolRoot, protocol_module)):
print("Can't find protocol module file " + protocol_module + ".")
sys.exit()
# import and initialize source
def lp(labbook, protocol):
return str(ansi.magenta + labbook.id + ansi.cyan + '@' + ansi.magenta +
protocol.PROTOCOLID + ansi.clear)
################################################################################
# Argument Processing
################################################################################
items = []
for item_list in args.lp:
if '@' not in item_list[0]:
print("Invalid labbook_id@protocol_id format.")
sys.exit()
labbook_id, protocol_id = item_list[0].split('@')
if not plbCore.validID(labbook_id):
print("Labbook id '" + labbook_id + "' contains invalid characters.")
sys.exit()
if not plbCore.validID(protocol_id):
print("Labbook id '" + protocol_id + "' contains invalid characters.")
sys.exit()
for esid in item_list[1:]:
if '.' in esid:
eid, sid = esid.split('.')
else:
eid = esid
sid = None
if not plbCore.validID(eid):
print("Experiment id '" + eid + "' contains invalid characters.")
sys.exit()
if sid != "" and not plbCore.validID(eid):
def create(protocol_id):
# verify id
if not plbCore.validID(protocol_id):
print("'" + protocol_id + "' contains invalid characters")
sys.exit()
# check set/sam description files
file_setdesc = os.path.join(root, protocol_id + "_SET_DESC.csv")
file_samdesc = os.path.join(root, protocol_id + "_SAMPLE_DESC.csv")
print("Checking for " + ansi.green + file_setdesc + ansi.clear + ": ",
end='',
flush=True)
if not os.path.isfile(file_setdesc):
print(ansi.red + "NOT FOUND" + ansi.clear)
sys.exit()
print(ansi.green + "OK" + ansi.clear)
print("Checking for " + ansi.green + file_samdesc + ansi.clear + ": ",
end='',
flush=True)
if not os.path.isfile(file_samdesc):
print(ansi.red + "NOT FOUND" + ansi.clear)
sys.exit()
print(ansi.green + "OK" + ansi.clear)
# check for existing module file
modfile = protocol_id + ".py"
modffn = os.path.join(plbProtocolRoot, modfile)
if os.path.isfile(modffn):
print("There is already a module for " + protocol_id + "...")
print("Exiting.")
sys.exit()
# load and check records
setdesc = pd.read_csv(file_setdesc, index_col=None, header=0)
samdesc = pd.read_csv(file_samdesc, index_col=None, header=0)
if len(setdesc) == 0:
print("Set description is empty... Nothing to do.")
sys.exit()
# empty sample description is okay.
# dummy labbook object
try:
dummy_labbook = pyLabbook(
id="dummy",
root=plbRoot,
repositoryPath=plbRoot,
sheetFormat='csv',
databasePath=plbRoot,
databaseFile="temp.sqlite3",
databaseFormat='SQLITE3',
)
except Exception as e:
print("Error initializing placeholder pyLabbook: " + str(e))
sys.exit()
# initialize empty protocol
try:
protocol = pyProtocol(dummy_labbook)
except Exception as e:
print("Error initializing empty pyProtocol: " + str(e))
sys.exit()
protocol.PROTOCOLID = protocol_id
# confirm set/sample description columns
desc_columns = pd.Series(protocol.setDesc().columns.sort_values().tolist())
if not (pd.Series(setdesc.columns.sort_values().tolist())
== desc_columns).all():
print("Set description file contains invalid or missing columns...")
print("Exiting.")
sys.exit()
if not (pd.Series(samdesc.columns.sort_values().tolist())
== desc_columns).all():
print("Sample description file contains invalid or missing columns...")
print("Exiting.")
sys.exit()
# TODO: need a more universal way to deal with these casting issues
# a little data formatting and type enforcement
for df in [setdesc, samdesc]:
# preliminary casting
df['notnull'] = df['notnull'].astype(str)
df['unique'] = df['unique'].astype(str)
df['primary_key'] = df['primary_key'].astype(str)
for i, r in df.iterrows():
# casting for more complicated issues... --> np.nan's...
# set all pd.null's to "" and let SQL engine figure it out.
if pd.isnull(r['default']): df.loc[i, 'default'] = ""
# convert ALL string values manually to avoid "nan"
# TODO: more elegant solution for this...
for strcol in ['description', 'name', 'type']:
if pd.isnull(r[strcol]): df.loc[i, strcol] = ""
# true/false
for boolcol in ['notnull', 'unique', 'primary_key']:
if str(r[boolcol]).upper() == "TRUE" or r[boolcol] == "1":
df.loc[i, boolcol] = True
elif str(r[boolcol]).upper() == "FALSE" or r[boolcol] == "0":
df.loc[i, boolcol] = False
else:
print("Invalid boolean identifier: " + r[boolcol] + ".")
sys.exit()
# last minute formatting
df['type'] = df['type'].str.upper()
# build protocol
protocol.id = protocol_id
for table in ['set', 'sam']:
if table == 'set':
df = setdesc
addColumn = protocol.addSetColumn
print("Building set description:")
if table == 'sam':
df = samdesc
addColumn = protocol.addSamColumn
print("Building sample description:")
if len(df) < 1:
print("\tNo columns to process...")
continue
for i, r in df.iterrows():
# build keyword args and use addSetColumn
kwa = {}
for k in desc_columns:
# special case for default
if k == 'default' and r[k] == '':
kwa[k] = None
# otherwise just map
else:
kwa[k] = r[k]
print("\t" + ansi.magenta + str(r['name']) + ansi.clear + ": ",
end='',
flush=True)
try:
addColumn(**kwa)
except Exception as e:
print(ansi.red + "ERROR" + ansi.clear)
print(str(e))
sys.exit()
print(ansi.green + "OK" + ansi.clear)
print("Serializing...")
serialized = protocol.exportSerialized()
print("Writing to " + ansi.green + modffn + ansi.clear + ": ",
end="",
flush=True)
try:
with open(modffn, 'w') as fh:
fh.write(serialized)
except Exception as e:
print(ansi.red + "ERROR" + ansi.clear)
print(str(e))
sys.exit()
print(ansi.green + "OK" + ansi.clear)
print("Done.")
def initialize(protocol_id):
# verify id
if not plbCore.validID(protocol_id):
print("'" + protocol_id + "' contains invalid characters")
sys.exit()
# check for no module
modfile = protocol_id + ".py"
modffn = os.path.join(plbProtocolRoot, modfile)
if os.path.isfile(modffn):
print("There is already a module for " + protocol_id + "...")
print("Exiting.")
sys.exit()
# dummy labbook object
try:
dummy_labbook = pyLabbook(
id="dummy",
root=plbRoot,
repositoryPath=plbRoot,
sheetFormat='csv',
databasePath=plbRoot,
databaseFile="temp.sqlite3",
databaseFormat='SQLITE3',
)
except Exception as e:
print("Error initializing placeholder pyLabbook: " + str(e))
sys.exit()
# initialize empty protocol
try:
protocol = pyProtocol(dummy_labbook)
except Exception as e:
print("Error initializing empty pyProtocol: " + str(e))
sys.exit()
# get empty set and sample descriptions
set_desc = pd.DataFrame(columns=protocol.setDesc().columns)
sam_desc = pd.DataFrame(columns=protocol.samDesc().columns)
# save csv format only
outfile_setdesc = os.path.join(root, protocol_id + "_SET_DESC.csv")
outfile_samdesc = os.path.join(root, protocol_id + "_SAMPLE_DESC.csv")
if os.path.isfile(outfile_setdesc):
print("There is already a set description called " + outfile_setdesc +
"...")
sys.exit()
print("Writing to " + ansi.green + outfile_setdesc + ansi.clear + ": ",
end='',
flush=True)
try:
set_desc.to_csv(outfile_setdesc, index=False, header=True)
except Exception as e:
print(ansi.red + "ERROR" + ansi.clear)
print(str(e))
sys.exit()
print(ansi.green + "OK" + ansi.clear)
if os.path.isfile(outfile_samdesc):
print("There is already a set description called " + outfile_samdesc +
"...")
sys.exit()
print("Writing to " + ansi.green + outfile_samdesc + ansi.clear + ": ",
end='',
flush=True)
try:
sam_desc.to_csv(outfile_samdesc, index=False, header=True)
except Exception as e:
print(ansi.red + "ERROR" + ansi.clear)
print(str(e))
sys.exit()
print(ansi.green + "OK" + ansi.clear)
print("Fill out these forms and use the " + ansi.br_cyan + "create" +
ansi.clear + " command to create " + protocol_id + ".")
except Exception as e:
print("Error deleting " + modffn + ": " + str(e))
sys.exit()
print("Successfully deleted " + modffn + ".")
print("Done")
#------------------------------------------------------------------------------#
################################################################################
# Argument Processing
################################################################################
# dict of available commands mapped to their function refs
command_map = {
'initialize': initialize,
'create': create,
'delete': delete,
}
# validate command
if not args.command in command_map.keys():
print("Unrecognized command '" + args.command + "'...")
sys.exit()
# validate labbook
if not plbCore.validID(args.protocol):
print("'" + args.protocol + "' contains invalid characters")
sys.exit()
# call command
command_map[args.command](args.protocol)
#