def getFiles_withoutPartner(self):
for libid in sorted(self.__libdict):
directionlist = self.__libdict[libid].keys()
if len(directionlist) == 2 and 'R1-R2' not in directionlist:
# build sets and subtract them from each other
loneset = set([i for i in self.__libdict[libid]['R1']])
ltwoset = set([i for i in self.__libdict[libid]['R2']])
onlyone = loneset - ltwoset
onlytwo = ltwoset - loneset
for i, checkset in enumerate((onlyone, onlytwo)):
if i == 0:
read = 'R1'
else:
read = 'R2'
if len(checkset) != 0:
# build a new list with the difference
temp = []
[temp.extend(self.__libdict[libid][read][i]) for i in checkset]
self.__libdict[libid]['{0}-only'.format(read)]['tracks'] = temp
# remove the tracks from the original entry
for i in checkset:
del self.__libdict[libid][read][i]
if len(self.__libdict[libid][read]) == 0:
del self.__libdict[libid][read]
self.show_log('info', '{0}.{1} - The following Tracks are only in {2} and will be processed separately'.format(self.__classname, getframe().f_code.co_name, read))
for name in [i.filename for i in temp]:
self.show_log('info', '{0}.{1} - {2}'.format(self.__classname, getframe().f_code.co_name, name))
sleep(2)
def buildSGEobject(self, dbinst = None, separate = True):
for libid in sorted(self.__libdict):
for direction in self.__libdict[libid]:
# sum up the number of fileobjects
fileobjects = []
[fileobjects.extend(i) for i in self.__libdict[libid][direction].viewvalues()]
fileobjects = sorted(fileobjects, key = attrgetter('track'))
sample = '-'.join(sorted(set(fileobjects[0].sample)))
sample = sample.replace(' ', '_')
length = len(fileobjects)
if length > 1:
self.show_log('info', '{0}.{1} - Lib {2}; Direction: {3}; Files: {4}; status: merged and transferred'.format(self.__classname, getframe().f_code.co_name, libid, direction, length))
else:
self.show_log('info', '{0}.{1} - Lib {2}; Direction: {3}; Files: {4}; status: not merged but transferred'.format(self.__classname, getframe().f_code.co_name, libid, direction, length))
for obj in fileobjects:
self.show_log('info', '{0}.{1} - File {2}'.format(self.__classname, getframe().f_code.co_name, obj.filename))
trackstring = '-'.join(['-'.join(i.track) for i in fileobjects])
# inputfiles for MergeFastqSGE object have to be sorted!
sgeinst = MergeFastqSGE(self.__zipped, self._log, False, Information.EMAILADRESS, fileobjects, 1, 1, cores = 6)
namestring = '{0}_{1}_Track-{2}_{3}'.format(libid, sample, trackstring, direction)
if separate:
sgeinst.moduledir = Information.MAINDIR
sgeinst.set_logfilename_line(namestring)
sgeinst.set_qstatname_line(namestring)
sgeinst.set_qsubfile(namestring)
sgeinst.build_outputfiles()
sgeinst.outputfiles[0].querydb = dbinst
sgeinst.outputfiles[0].set_information()
sgeinst.build_commandline()
sgeinst.set_qsubscript()
self.__sgelist.append(sgeinst)
sleep(2)
def _py_backwards_walrus(name, value):
frame = getframe(1)
frame.f_locals[name] = value
ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(frame),
ctypes.c_int(0))
del frame
return value
def checkFiles(self, filelist):
if isinstance(filelist, NoneType):
return []
good, bad = check_Fileslist(filelist)
goodfiles = [i for i in good if i.endswith(('fastq.gz', 'fastq', 'fq.gz', 'fq'))]
badfiles = [i for i in good if not i.endswith(('fastq.gz', 'fastq', 'fq.gz', 'fq'))]
badfiles.extend(bad)
for i in goodfiles:
self.show_log('info', "{0}.{1} - valid file: {2}".format(self.__classname, getframe().f_code.co_name, i))
for i in badfiles:
self.show_log('warning', "{0}.{1} - check file: {2}".format(self.__classname, getframe().f_code.co_name, i))
return goodfiles
def filter(self, record):
if self._line:
record.lineno = self._line
else:
# record.name = getframe(stack).f_code.co_name
record.lineno = getframe(self._stack).f_lineno
return True
def calculate_LineNumbers(self):
call = Popen(['zcat', self._filename], stdout=PIPE)
countcall = Popen(['wc', '-l'], stdin=call.stdout, stdout=PIPE)
call.stdout.close()
count = int(countcall.communicate()[0])/4
self.show_log('debug', "{0}.{1} - reads - {2}: {3}".format(self.__classname, getframe().f_code.co_name, self._filename, count))
return count
def getID(self):
# this function will print out the ai_id of the caller (or his parent, maybe)
i = 2
errored_once = False
while True:
try:
i += 1
try:
frame = getframe(i)
except ValueError:
i = 0
if errored_once:
return
errored_once = True
continue
f_locals = frame.f_locals
try:
if issubclass(f_locals['self'].__class__, ai.BareAI):
ai_id = f_locals['self'].ai_id
return ai_id
except KeyError:
pass
finally:
try:
pass
except:
pass
def create_manager(name, index):
frame = getframe(1) # get caller frame
kwargs = dict(Builder.__dict__)
kwargs['__module__'] = frame.f_locals.get('__name__', '')
manager = type(name, (Builder,), kwargs)
manager = six.add_metaclass(abc.ABCMeta)(manager)
manager.index = index
return manager
def _(request=None, *args, **kwargs):
frame = getframe(1)
stub.logger.handle(LogRecord(
stub.logger.name, level, frame.f_code.co_filename,
frame.f_lineno, '[stub|%s][request|%s][kwargs|%s]',
(stub_name, request, kwargs), None, stub_name
))
return rpc(request, *args, **kwargs)
def build_warning(cls, name, code, message):
frame = getframe(1) # get caller frame
warning = type(
name, (Warning,),
{'code': cls.index + code, 'message': message,
'__module__': frame.f_locals.get('__package__', '')}
)
setattr(cls, name, warning)
cls.register(warning)
def build_error(cls, name, code, message):
frame = getframe(1) # get caller frame
error = type(
name, (Error,),
{'code': cls.index + code, 'message': message,
'__module__': frame.f_locals.get('__package__', '')}
)
setattr(cls, name, error)
cls.register(error)
def _(request=None, conn=None, broadcaster=None, **kwargs):
frame = getframe(1)
stub.logger.handle(LogRecord(
stub.logger.name, level, frame.f_code.co_filename,
frame.f_lineno,
'[stub|%s][request|%r][kwargs|%r]',
(stub_name, str(request), str(kwargs)), None, stub_name
))
return rpc(request, conn, broadcaster, **kwargs)
def add_error(cls, name, code, message):
frame = getframe(1) # get caller frame
error = type(
name, (Error,),
{'code': cls.index + code, 'message': message,
'__module__': frame.f_locals.get('__name__', '')}
)
cls.add(name, error)
cls.register(error)
return error
def _verify_project_id_exists(self):
if not self.project_id:
caller_name = 'UNKNOWN'
try:
caller_name = sys.getframe(1).f_code.co_name
except Exception as ex:
caller_name = inspect.stack()[1][3]
raise ApiError(
'Project ID not set on API connection and is required by {}().'
.format(caller_name))
def add_warning(cls, name, code, message):
frame = getframe(1) # get caller frame
warning = type(
name, (Warning,),
{'code': cls.index + code, 'message': message,
'__module__': frame.f_locals.get('__name__', '')}
)
cls.add(name, warning)
cls.register(warning)
return warning
def create_manager(cls, name, index):
frame = getframe(1) # get caller frame
kwargs = dict(ErrorManager.__dict__)
kwargs['__module__'] = frame.f_locals.get('__name__', '')
manager = type(name, (ErrorManager,), kwargs)
manager.index = index
manager.codes = {}
manager.exceptions = {}
manager.managers = {}
cls.add(name, manager)
return manager
def trace_call_comment(depth=1):
frame = getframe(depth)
if frame.f_code.co_name != 'circuit':
frame = getframe(depth + 1)
# Search in code
code = []
notes = []
try:
code = getsourcelines(frame.f_code)[0]
code_line = frame.f_lineno - frame.f_code.co_firstlineno
except OSError:
if hasattr(builtins, 'code'):
code = builtins.code.split('\n')
code_line = frame.f_lineno - 1
if len(code):
notes = inspect_comments(code, code_line, code_line)
return notes
def SetLogger(package, stackNum=0):
if not myLOGGER:
return _setNullLogger()
funcName = getframe(stackNum + 1).f_code.co_name
funcName_prev = getframe(stackNum).f_code.co_name
if funcName == '<module>': funcName = '__main__'
# - tracciamo la singola funzione oppure modulo oppure libreria od altro
if '!ALL!' in modulesToLog:
LOG_LEVEL = logging.DEBUG
else:
LOG_LEVEL = None # default
fullPkg = (package + funcName).lower()
for moduleStr in modulesToLog:
if moduleStr.lower() in fullPkg:
LOG_LEVEL = logging.DEBUG
if False:
print(__name__, 'package..................', package)
print(__name__, 'funcName.................', funcName)
print(__name__, 'funcName_prev............', funcName_prev)
print(__name__, 'LOG_LEVEL................', LOG_LEVEL)
print()
if LOG_LEVEL:
logger = logging.getLogger(package)
logger.setLevel(LOG_LEVEL)
# caller = inspect.stack()[stackNum]
# dummy, programFile, lineNumber, funcName, lineCode, rest = caller
logger.info('\n')
logger.info('{TARGET}......called by:{CALLER}'.format(
TARGET=funcName_prev, CALLER=_GetCaller(stackNum + 2)))
else:
logger = _setNullLogger()
return logger
def handle_stackframe_without_leak():
cat = ["meow"]
frame = sys.getframe(0)
print frame
try:
print inspect.getframeinfo(frame)
# do something with the frame
finally:
del frame
def handle_stackframe_without_leak():
cat = ["meow"]
frame = sys.getframe(0)
print frame
try:
print inspect.getframeinfo( frame )
# do something with the frame
finally:
del frame
def fileWriteLine(filename, line, source=DEFAULT_MODULE_NAME, debug=False):
try:
with open(filename, "w") as fd:
fd.write(str(line))
msg = "Wrote"
result = 1
except OSError as err:
print("[%s] Error %d: Unable to write a line to file '%s'! (%s)" % (source, err.errno, filename, err.strerror))
msg = "Failed to write"
result = 0
if debug or forceDebug:
print("[%s] Line %d: %s '%s' to file '%s'." % (source, getframe(1).f_lineno, msg, line, filename))
return result
def checkFiles(self):
filelist = self.__options.files
if isinstance(filelist, NoneType):
return
good, bad = check_Fileslist(filelist)
good = [i for i in good if i.endswith('fastq.gz') or i.endswith('fq.gz')]
# for i in good:
# self._logger.debug("{0}.{1} - valid files: {2}".format(self.__classname, getframe().f_code.co_name, i))
for i in bad:
self.show_log('warning', "{0}.{1} - check files: {2}".format(self.__classname, getframe().f_code.co_name, i))
self._filelist = good
def exists(x, where="local"):
""" Exists
Description
-----------
Checks if an object exists by the name of the object
Parameters
----------
x : str
The name of an object as a string
where : str, "local" (default), "global", or "builtin"
Where to search for the object. If not one of the three above, raises
and exception
Returns
-------
If object is found, True, otherwise False
References
----------
Adapted from: https://stackoverflow.com/a/6386015/12126576
"""
import __builtin__
import sys
if where == "local":
res = x in sys.getframe(1).f_locals
elif where == "global":
res = x in sys.getframe(1).f_globals
elif where == "builtin":
res = x in vars(__builtin__)
else:
raise Warning("`where` should be one of: 'local', 'global', 'builtin'")
return (res)
def _get_registry(db_name):
"""Helper method to get the registry for a `db_name`."""
from odoo.modules.registry import Registry
if isinstance(db_name, string_types):
db = Registry(db_name)
elif isinstance(db_name, Registry):
db = db_name
else:
import sys
caller = sys.getframe(1).f_code.co_name
raise TypeError('"%s" requires a string or a Registry' % caller)
return db
def fileReadLine(filename, default=None, source=DEFAULT_MODULE_NAME, debug=False):
line = None
try:
with open(filename, "r") as fd:
line = fd.read().strip().replace("\0", "")
msg = "Read"
except OSError as err:
if err.errno != ENOENT: # ENOENT - No such file or directory.
print("[%s] Error %d: Unable to read a line from file '%s'! (%s)" % (source, err.errno, filename, err.strerror))
line = default
msg = "Default"
if debug or forceDebug:
print("[%s] Line %d: %s '%s' from file '%s'." % (source, getframe(1).f_lineno, msg, line, filename))
return line
def update_File(self):
readcount, md5sum = None, None
self.show_log('info', "{0}.{1} - updating: {2}".format(self.__classname, getframe().f_code.co_name, self._filename))
trackstatus = self.get_Track_Status()
if trackstatus == -1:
self.show_log('warning', "{0}.{1} - file in database - track problems: {2}".format(self.__classname, getframe().f_code.co_name, self._filename))
return False
if trackstatus in (1,2,5):
self.__trackstatus = 3
self.__db.update_Trackstatus_into_Tracks(self.__trackid, self.__trackstatus)
self.__updatetrackstatus = True
elif trackstatus == 3:
self.__updatetrackstatus = False
elif trackstatus == 4:
self.__updatetrackstatus = False
if self.__ln:
readcount = self.calculate_LineNumbers()
if self.__md5:
md5sum = self.calculate_MD5SUM()
self.__db.update_FQfile_into_FQstorage(basename(self._filename), readcount, md5sum, self.__publish, self.__waspublished, self.__casava, self.__comment, self.__date)
self.show_log('info', "{0}.{1} - done: {2}".format(self.__classname, getframe().f_code.co_name, self._filename))
def fileReadXML(filename,
default=None,
source=DEFAULT_MODULE_NAME,
debug=False):
dom = None
try:
with open(
filename, "r"
) as fd: # This open gets around a possible file handle leak in Python's XML parser.
try:
dom = parse(fd).getroot()
msg = "Read"
except ParseError as err:
fd.seek(0)
content = fd.readlines()
line, column = err.position
print("[%s] XML Parse Error: '%s' in '%s'!" %
(source, err, filename))
data = content[line - 1].replace("\t", " ").rstrip()
print("[%s] XML Parse Error: '%s'" % (source, data))
print("[%s] XML Parse Error: '%s^%s'" %
(source, "-" * column, " " * (len(data) - column - 1)))
except Exception as err:
print("[%s] Error: Unable to parse data in '%s' - '%s'!" %
(source, filename, err))
except (IOError, OSError) as err:
if err.errno == ENOENT: # ENOENT - No such file or directory.
print("[%s] Warning: File '%s' does not exist!" %
(source, filename))
else:
print("[%s] Error %d: Opening file '%s'! (%s)" %
(source, err.errno, filename, err.strerror))
except Exception as err:
print("[%s] Error: Unexpected error opening/parsing file '%s'! (%s)" %
(source, filename, err))
print_exc()
if dom is None:
if default and isinstance(default, str):
dom = fromstring(default)
msg = "Default (XML)"
elif default and isinstance(default, Element):
dom = default
msg = "Default (DOM)"
else:
msg = "Failed to read"
if debug or forceDebug:
print("[%s] Line %d: %s from XML file '%s'." %
(source, getframe(1).f_lineno, msg, filename))
return dom
def fileReadLines(filename, default=None, source=DEFAULT_MODULE_NAME, debug=False):
lines = None
try:
with open(filename, "r") as fd:
lines = fd.read().splitlines()
msg = "Read"
except OSError as err:
if err.errno != ENOENT: # ENOENT - No such file or directory.
print("[%s] Error %d: Unable to read lines from file '%s'! (%s)" % (source, err.errno, filename, err.strerror))
lines = default
msg = "Default"
if debug or forceDebug:
length = len(lines) if lines else 0
print("[%s] Line %d: %s %d lines from file '%s'." % (source, getframe(1).f_lineno, msg, length, filename))
return lines
def fileWriteLines(filename, lines, source=DEFAULT_MODULE_NAME, debug=False):
try:
with open(filename, "w") as fd:
if isinstance(lines, list):
lines.append("")
lines = "\n".join(lines)
fd.write(lines)
msg = "Wrote"
result = 1
except OSError as err:
print("[%s] Error %d: Unable to write %d lines to file '%s'! (%s)" % (source, err.errno, len(lines), filename, err.strerror))
msg = "Failed to write"
result = 0
if debug or forceDebug:
print("[%s] Line %d: %s %d lines to file '%s'." % (source, getframe(1).f_lineno, msg, len(lines), filename))
return result
def process_web_queue():
"""
处理web请求
"""
global web_req_queue
while True:
data = web_req_queue.get()
# print("接收20001端口，命令："+str(data))
# print(type(data))
msg_type = data['type']
serial = data['serialnum']
if msg_type == 'download':
serial_code = data['serialnum']
for client in factory.clients:
# print(client)
if not hasattr(client, 'serial_code'):
continue
if client.serial_code == serial_code:
msg_option = data['option']
if msg_option == 'active':
client.send(json.dumps(data))
elif msg_option == 'updateconf':
# 从数据库获取设备配置信息发送给硬件
# 0905170211: 12:58
print("updateconf")
configure = mdb.getdeviceconf(serial)
if configure != False:
data['message'] = configure
else:
print(
"when using configure from webserver, quary database has fault in %s : %d"
) % (__file__, sys.getframe().f_lineno)
print(data)
client.send(json.dumps(data))
# end
else:
print("无法识别的Web申请...")
break
else:
log.msg('没有合适的网关')
else:
log.msg('解析web请求时，遇到无法识别的数据包类型')
def getID():
# this function will print out the template_id of the caller (or his parent, maybe)
i = 2
errored_once = False
while True:
i += 1
try:
frame = getframe(i)
except ValueError:
i = 0
if errored_once:
return
errored_once = True
continue
f_locals = frame.f_locals
try:
if issubclass(f_locals['self'].__class__, template.Template):
template_id = id(f_locals['self'])
return template_id
except KeyError:
pass
def createFilelist(self):
good, bad = [], []
# add files to filelist
for i in self.__dirlist:
self._filelist.extend(add_FilestoList_recursive(i, [], ('fastq.gz', 'fq.gz')))
# make unique
for name in set(self._filelist):
if self.__filereg.match(basename(name)):
good.append(name)
else:
bad.append(name)
for i in bad:
self.show_log('warning', "{0}.{1} - check files: {2}".format(self.__classname, getframe().f_code.co_name, i))
if len(good) != 0:
self.show_log('info', "{0}.{1} - files for database: {2}".format(self.__classname, getframe().f_code.co_name, len(self._filelist)))
self._filelist = good
self.__ready = True
else:
self.show_log('info', "{0}.{1} - no files for database: {2}".format(self.__classname, getframe().f_code.co_name, len(self._filelist)))
self.__ready = False
def check_library_delivery_state(self):
librarylist = self.__db.query_Library_with_Status(9)
if len(librarylist) != 0:
self.show_log('info', "{0}.{1} - Number of libraries to check - {2}".format(self.__classname, getframe().f_code.co_name, len(librarylist)))
for libentry in librarylist:
fqcount = 0
libid = libentry['ID']
self.show_log('info', "{0}.{1} - Checking library - {2}".format(self.__classname, getframe().f_code.co_name, libid))
tracklist = self.__db.query_Tracks_with_libid(libid)
self.show_log('info', "{0}.{1} - Tracks from library '{3}' - {2}".format(self.__classname, getframe().f_code.co_name, len(tracklist), libid))
for trackentry in tracklist:
trackstatus = trackentry['TRACKSSTATUS_ID']
if trackstatus in (1,2,4,5):
continue
fqfilelist = self.__db.query_FQstorage_with_trackid(trackentry['ID'])
dateinst = datetime.now()
datestring = '{0}-{1}-{2}'.format(dateinst.year, dateinst.month, dateinst.day)
for fqentry in fqfilelist:
fqcount += 1
self.__db.update_FQfile_into_FQstorage(fqentry['FILE'], None, None, 1, 0, None, None, datestring)
self.show_log('info', "{0}.{1} - FQ-files from library '{3}' set to be published - {2}".format(self.__classname, getframe().f_code.co_name, fqcount, libid))
self.__db.update_Status_into_Library(libid, 6)
self.show_log('info', "{0}.{1} - Done library - {2}".format(self.__classname, getframe().f_code.co_name, libid))
def find_caller_frame(cls):
"""
Find the stack frame of the logging caller.
:returns: 'frame' object such as from sys._getframe()
"""
# Go up through stack frames until we find the first with a
# type(f_locals.self) of logging.Logger. This should be the start
# of the frames internal to logging.
frame: 'frame' = getframe(0)
while frame:
if isinstance(frame.f_locals.get('self'), logging.Logger):
frame = frame.f_back # Want to start on the next frame below
break
frame = frame.f_back
# Now continue up through frames until we find the next one where
# that is *not* true, as it should be the call site of the logger
# call
while frame:
if not isinstance(frame.f_locals.get('self'), logging.Logger):
break # We've found the frame we want
frame = frame.f_back
return frame
def on_text_message_removed_by_admin(self, data):
self.call(getframe(0).f_code.co_name, objects.Event(data["o"]).Event)
def whoami(): from sys import _getframe as getframe return getframe(1).f_code.co_name
def check_library_sequencing_amount(self):
librarylist = self.__db.query_Library_with_Status(3)
if len(librarylist) != 0:
self.show_log('info', "{0}.{1} - Number of libraries to check - {2}".format(self.__classname, getframe().f_code.co_name, len(librarylist)))
for libentry in librarylist:
readcount = 0
libid = libentry['ID']
# print libid
# print libentry['REQUESTED_SEQUENCING']
# print type(libentry['REQUESTED_SEQUENCING'])
# read_req = libentry['REQUESTED_SEQUENCING'].to_integral_value()
read_req = libentry['REQUESTED_SEQUENCING']
self.show_log('info', "{0}.{1} - Checking library - {2}".format(self.__classname, getframe().f_code.co_name, libid))
# library should have a read number, otherwise script can't work
if isinstance(read_req, NoneType):
self.show_log('warning', "{0}.{1} - Reads requested for library '{2}' set to 0 - CHANGE!".format(self.__classname, getframe().f_code.co_name, libid))
self.__zeroread += 1
continue
self.show_log('info', "{0}.{1} - Reads requested for library '{2}' - {3}".format(self.__classname, getframe().f_code.co_name, libid, read_req))
tracklist = self.__db.query_Tracks_with_libid(libid)
# no tracks -> no fastq files -> no check
if len(tracklist) == 0:
self.show_log('warning', "{0}.{1} - No tracks for library '{2}' - skipping".format(self.__classname, getframe().f_code.co_name, libid))
self.__zerotrack += 1
continue
self.show_log('info', "{0}.{1} - Number of tracks from library '{2}' - {3}".format(self.__classname, getframe().f_code.co_name, libid, len(tracklist)))
# track has one entry if single read, if paired end two
# check recipe what kind of track it is
no_count = False
for trackentry in tracklist:
status = trackentry['TRACKSSTATUS_ID']
if status in (1, 2, 5):
no_count = True
self.show_log('info', "{0}.{1} - Library '{2}' has at least one track without reads - skipping".format(self.__classname, getframe().f_code.co_name, libid))
self.__zerofqfile += 1
break
if no_count:
continue
for trackentry in tracklist:
trackid = trackentry['ID']
status = trackentry['TRACKSSTATUS_ID']
self.show_log('debug', "{0}.{1} - Processing track '{2}' from library '{3}' ".format(self.__classname, getframe().f_code.co_name, trackid, libid))
if status in (1, 2, 5):
self.show_log('debug', "{0}.{1} - Track '{2}' from library '{3}' has no FQ-file(s)".format(self.__classname, getframe().f_code.co_name, trackid, libid))
self.__zerofqfile += 1
break
elif status == 4:
self.show_log('debug', "{0}.{1} - Track '{2}' from library '{3}' failed".format(self.__classname, getframe().f_code.co_name, trackid, libid))
continue
recipe = trackentry['RECIPE']
fqfilelist = self.__db.query_FQstorage_with_trackid(trackentry['ID'])
fqfiles = len(fqfilelist)
if fqfiles > 2:
self.show_log('error', "{0}.{1} - Track '{2}' from library '{3}' - more than 2 FQ-files".format(self.__classname, getframe().f_code.co_name, trackid, libid))
continue
elif fqfiles == 2 and recipe == 'PE':
readcount += int(fqfilelist[0]['READCOUNT'])
elif fqfiles == 1 and recipe == 'SR':
readcount += int(fqfilelist[0]['READCOUNT'])
else:
self.show_log('error', "{0}.{1} - Check track '{2}' from library '{3}' - Recipe '{4}' does not match fq-file amount '{5}'".format(self.__classname, getframe().f_code.co_name, trackid, libid, recipe, fqfiles))
continue
readcount = readcount / 1000000
read_req = int(read_req)
lower_limit = read_req - (read_req * 0.1)
print libid, read_req, lower_limit, readcount
self.show_log('info', "{0}.{1} - Library(ies) with read requested set to 0 - {2}".format(self.__classname, getframe().f_code.co_name, self.__zeroread))
self.show_log('info', "{0}.{1} - Library(ies) with no tracks - {2}".format(self.__classname, getframe().f_code.co_name, self.__zerotrack))
self.show_log('info', "{0}.{1} - Library(ies) with FQ-files not done - {2}".format(self.__classname, getframe().f_code.co_name, self.__zerofqfile))
def _property_called_name(self):
property_called_name = self.__last_property_called__ or getframeinfo(getframe(2))[2]
self.__last_property_called__ = None
return property_called_name
def on_user_typing_end(self, data):
self.call(getframe(0).f_code.co_name, objects.Event(data["o"]).Event)
def on_chat_tipping_disabled(self, data):
self.call(getframe(0).f_code.co_name, objects.Event(data["o"]).Event)
def on_chat_view_only_enabled(self, data):
self.call(getframe(0).f_code.co_name, objects.Event(data["o"]).Event)
def type_date(self):
datestring = self.__options.date
if isinstance(datestring, NoneType):
return
error = False
try:
datelist = datestring.split('-')
except AttributeError:
error = True
if len(datelist) != 3:
error = True
try:
datelist = [int(i) for i in datelist]
except ValueError:
error = True
try:
datetime(year=datelist[0], month=datelist[1], day=datelist[2])
except ValueError:
if datestring != '0000-00-00':
error = True
except IndexError:
error = True
if error:
self.show_log('error' ,"{0}.{1} - not valid date format: {2}".format(self.__classname, getframe().f_code.co_name, datestring))
exit(1)
self.show_log('debug', "{0}.{1} - valid date format: {2}".format(self.__classname, getframe().f_code.co_name, datestring))
self.__date = datestring
def __methodCalled__(self, *args, **kargs):
property = getframeinfo(getframe(1))[2]
return self._property_called(property, args, kargs)
def checkDirectory(self):
dirlist = self.__options.directory
if isinstance(dirlist, NoneType):
return
good, bad = check_Directorylist(dirlist)
for i in good:
self.show_log('debug', "{0}.{1} - valid directory: {2}".format(self.__classname, getframe().f_code.co_name, i))
for i in bad:
self.show_log('warning', "{0}.{1} - check directory: {2}".format(self.__classname, getframe().f_code.co_name, i))
self.__dirlist = good
def check_inputfiles(self, optionlist = []):
if len(self._inputfiles) > 2:
self.show_log('error', "{0}.{1} - not more than 2 files excepted. Here {2} files".format(self._classname, getframe().f_code.co_name, len(self._inputfiles)))
exit(2)
elif '--paired' in optionlist and len(self._inputfiles) == 1:
self.show_log('error', "{0}.{1} - --paired set but only one file".format(self._classname, getframe().f_code.co_name, len(self._inputfiles)))
exit(2)
elif len(self._inputfiles) == 2:
if not '--paired' in optionlist:
self.show_log('error', "{0}.{1} - --paired is not set but 2 files".format(self._classname, getframe().f_code.co_name, len(self._inputfiles)))
exit(2)
r1 = [i for i in self._inputfiles if i.read == ['R1']]
r2 = [i for i in self._inputfiles if i.read == ['R2']]
if len(r1) == 1 and len(r2) == 1 and not self.inputfiles[0].read == ['R1']:
self._inputfiles[0] = r1[0]
self._inputfiles[1] = r2[0]
self.show_log('info', "{0}.{1} - change order of inputfiles".format(self._classname, getframe().f_code.co_name))
elif len(r1) == 1 and len(r2) == 1 and self.inputfiles[0].read == ['R1']:
self.show_log('debug', "{0}.{1} - read direction correct".format(self._classname, getframe().f_code.co_name))
else:
self.show_log('debug', "{0}.{1} - read direction can't be defined. order of inputfiles remains untouched".format(self._classname, getframe().f_code.co_name))
def set_cpuvalue_line(self, system, cores = 1):
if len(self._inputfiles) == 1:
self.set_cpuline(system, 2)
self._cores = 2
elif len(self._inputfiles) == 2:
self.set_cpuline(system, 4)
self._cores = 4
elif len(self._inputfiles) >= 3:
self.show_log('warning', "{0}.{1} - not more than 2 files excepted. Here {2} files".format(self._classname, getframe().f_code.co_name, len(self._inputfiles)))
def __init__(self):
print("%s.%s() should be not called." % (self.__class__.__name__, sys.getframe().f_code.co_name))
def on_voice_chat_permission_open_to_everyone(self, data):
self.call(getframe(0).f_code.co_name, objects.Event(data["o"]).Event)
def on_voice_chat_permission_invite_only(self, data):
self.call(getframe(0).f_code.co_name, objects.Event(data["o"]).Event)
def on_avatar_chat_end(self, data):
self.call(getframe(0).f_code.co_name, objects.Event(data["o"]).Event)
def on_chat_unpin_announcement(self, data):
self.call(getframe(0).f_code.co_name, objects.Event(data["o"]).Event)
def on_chat_content_changed(self, data):
self.call(getframe(0).f_code.co_name, objects.Event(data["o"]).Event)
def on_invite_message(self, data):
self.call(getframe(0).f_code.co_name, objects.Event(data["o"]).Event)
def on_screen_room_end(self, data):
self.call(getframe(0).f_code.co_name, objects.Event(data["o"]).Event)
def default(self, data):
self.call(getframe(0).f_code.co_name, data)
def on_chat_host_transfered(self, data):
self.call(getframe(0).f_code.co_name, objects.Event(data["o"]).Event)
lower_limit = read_req - (read_req * 0.1)
print libid, read_req, lower_limit, readcount
self.show_log('info', "{0}.{1} - Library(ies) with read requested set to 0 - {2}".format(self.__classname, getframe().f_code.co_name, self.__zeroread))
self.show_log('info', "{0}.{1} - Library(ies) with no tracks - {2}".format(self.__classname, getframe().f_code.co_name, self.__zerotrack))
self.show_log('info', "{0}.{1} - Library(ies) with FQ-files not done - {2}".format(self.__classname, getframe().f_code.co_name, self.__zerofqfile))
'''
check library status for 'sequencing'
query reads requested for the libraries
if reads_requested - 10% is matched -> set libraries to finished/data delivered
if data_delivered -> publish reads
question: how find out if finished/data delivered -> via project? what if there are more than one project?
'''
if __name__ == '__main__':
db = Database(Information.DB_HOST, Information.DB_USER, Information.DB_PW, Information.DB)
logfile = '{0}{1}'.format(Information.LOGDIRECTORY, getLogfile(basename(getframe().f_code.co_filename.rstrip('.py'))))
mainlog = MainLogger(logfile)
db_main = DB_Maintain(db)
db_main.start_logging()
db.start_logging()
db.setConnection()
db_main.check_library_delivery_state()
#db_main.check_library_sequencing_amount()
db.commitConnection()
db.closeConnection()
def on_chat_removed_message(self, data):
self.call(getframe(0).f_code.co_name, objects.Event(data["o"]).Event)
def build_commandline(self, optionlist = []):
# trimmomatic PE -threads 10 -phred33 R1 R2 R1_P R1_U R2_P R2_U ILLUMINACLIP:/group/sequencing/Bfx/scripts/mathias/deep_seq_pipeline/files/trimmomatic_pe.fa:2:30:10:2:true LEADING:20 TRAILING:20 SLIDINGWINDOW:4:20 MINLEN:30
# trimmomatic SE -threads 10 -phred33 R1 R1_trim ILLUMINACLIP:/group/sequencing/Bfx/scripts/mathias/deep_seq_pipeline/files/trimmomatic_se.fa:2:30:10 LEADING:20 TRAILING:20 SLIDINGWINDOW:4:20 MINLEN:30
if not optionlist[0] in ('-phred33', '-phred64'):
self.show_log('error', "{0}.{1} - set phred quality score (either -phred33 or -phred64)".format(self._classname, getframe().f_code.co_name, len(self._inputfiles)))
exit(2)
commandline = []
if self._with_sgemodule:
commandline.append('trimmomatic')
else:
command = '{0} {1}{2}G {3} {4}'.format(Information.JAVAPATH, Information.JAVAMEM, self._ram, Information.JAR, self._progpath)
commandline.append(command)
if len(self._inputfiles) == 1:
commandline.append('SE')
else:
commandline.append('PE')
commandline.extend(('-threads', self.get_cpucommandcall()))
commandline.append(optionlist[0])
commandline.extend([i.completepath for i in self._inputfiles])
commandline.extend([i.completepath for i in self._outputfiles])
commandline.extend(optionlist[1:])
commandline = [i for i in commandline if i != '']
self._commandline = ' '.join(commandline)
def calculate_MD5SUM(self):
md5sum = md5(open(self._filename, 'rb').read()).hexdigest()
self.show_log('debug', "{0}.{1} - md5sum - {2}: {3}".format(self.__classname, getframe().f_code.co_name, self._filename, md5sum))
return md5sum
