class YouTubeClient(object):
"""
Network client for YouTube.
"""
## Derived from the ermrest iobox service client
def __init__(self, **kwargs):
self.baseuri = kwargs.get("baseuri")
o = urlparse.urlparse(self.baseuri)
self.scheme = o[0]
host_port = o[1].split(":")
self.host = host_port[0]
self.path = o.path
self.port = None
if len(host_port) > 1:
self.port = host_port[1]
self.cookie = kwargs.get("cookie")
self.client_secrets_file = kwargs.get("client_secrets_file")
self.client_oauth2_file = kwargs.get("client_oauth2_file")
self.catalog = PollingErmrestCatalog(self.scheme, self.host,
self.path.split('/')[-1],
{'cookie': self.cookie})
self.mail_server = kwargs.get("mail_server")
self.mail_sender = kwargs.get("mail_sender")
self.mail_receiver = kwargs.get("mail_receiver")
self.logger = kwargs.get("logger")
self.logger.debug('Delete YouTube Client initialized.')
"""
Send email notification
"""
def sendMail(self, subject, text):
if self.mail_server and self.mail_sender and self.mail_receiver:
retry = 0
ready = False
while not ready:
try:
msg = MIMEText('%s\n\n%s' % (text, mail_footer), 'plain')
msg['Subject'] = subject
msg['From'] = self.mail_sender
msg['To'] = self.mail_receiver
s = smtplib.SMTP(self.mail_server)
s.sendmail(self.mail_sender, self.mail_receiver.split(','),
msg.as_string())
s.quit()
self.logger.debug('Sent email notification.')
ready = True
except socket.gaierror as e:
if e.errno == socket.EAI_AGAIN:
time.sleep(100)
retry = retry + 1
ready = retry > 10
else:
ready = True
if ready:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error(
'%s' % str(traceback.format_exception(et, ev, tb)))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error(
'%s' % str(traceback.format_exception(et, ev, tb)))
ready = True
"""
Start the process for deleting files from YouTube
"""
def start(self):
try:
self.deleteFromYouTube()
except:
et, ev, tb = sys.exc_info()
self.logger.error('got unexpected exception "%s"' % str(ev))
self.logger.error('%s' %
str(traceback.format_exception(et, ev, tb)))
self.sendMail(
'FAILURE Delete YouTube: unexpected exception',
'%s\nThe process might have been stopped\n' %
str(traceback.format_exception(et, ev, tb)))
raise
"""
Get the YouTube Delete credentials
"""
def youtube_authenticated_service(self):
flow = InstalledAppFlow.from_client_secrets_file(
self.client_secrets_file, SCOPES)
storage = Storage(self.client_oauth2_file)
credentials = storage.get()
if credentials is None or credentials.invalid:
credentials = flow.run_console()
self.youtube = build(API_SERVICE_NAME,
API_VERSION,
credentials=credentials)
"""
Delete a video from YouTube
"""
def youtube_delete(self, youtube_uri):
res = False
try:
id = youtube_uri[youtube_uri.rfind('/') + 1:youtube_uri.index('?')]
self.logger.debug('Deleting YouTube video id="%s".' % (id))
self.youtube_authenticated_service()
if self.youtube is not None:
self.logger.debug(
'Authenticated to the YouTube delete service.')
response = self.youtube.videos().delete(id=id).execute()
self.logger.debug('Deleted response %s.' % (response))
res = True
else:
self.logger.debug(
'Authentication for deleting a YouTube video failed.')
except:
et, ev, tb = sys.exc_info()
self.logger.error('got YouTube exception "%s"' % str(ev))
self.logger.error('%s' %
str(traceback.format_exception(et, ev, tb)))
return res
"""
Delete videos from YouTube
"""
def deleteFromYouTube(self):
url = '/entity/Common:Delete_Youtube/Youtube_Deleted=FALSE/Processing_Status=in%20progress;Processing_Status::null::'
resp = self.catalog.get(url)
resp.raise_for_status()
files = resp.json()
fileids = []
for f in files:
fileids.append((f['YouTube_URI'], f['RID']))
self.logger.debug('Deleting from YouTube %d videos(s).' %
(len(fileids)))
for youtube_uri, rid in fileids:
try:
youtube_deleted = self.youtube_delete(youtube_uri)
if youtube_deleted == True:
self.logger.debug(
'SUCCEEDED deleted from YouTube the video with the URL: "%s".'
% (youtube_uri))
columns = ["Youtube_Deleted", "Processing_Status"]
columns = ','.join([urlquote(col) for col in columns])
url = '/attributegroup/Common:Delete_Youtube/RID;%s' % (
columns)
obj = {
'RID': rid,
'Youtube_Deleted': True,
'Processing_Status': 'success'
}
self.catalog.put(url, json=[obj])
self.logger.debug(
'SUCCEEDED updated the Common:Delete_Youtube table entry for the YouTube URL: "%s".'
% (youtube_uri))
else:
self.logger.debug(
'Failure in deleting from YouTube the video with the URL: "%s".'
% (youtube_uri))
self.sendMail(
'FAILURE Delete YouTube: YouTube Failure',
'The video "%s" could not be deleted from Youtube.' %
youtube_uri)
self.reportFailure(rid, 'YouTube Failure')
except Exception as e:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' %
str(traceback.format_exception(et, ev, tb)))
self.reportFailure(rid, str(e))
"""
Update the Delete_Youtube table with the ERROR status
"""
def reportFailure(self, rid, error_message):
"""
Update the Delete_Youtube table with the failure result.
"""
try:
columns = ["Processing_Status"]
columns = ','.join([urlquote(col) for col in columns])
url = '/attributegroup/Common:Delete_Youtube/RID;%s' % (columns)
obj = {'RID': rid, 'Processing_Status': '%s' % error_message}
self.catalog.put(url, json=[obj])
self.logger.debug(
'SUCCEEDED updated the Delete_Youtube table for the RID "%s" with the Processing_Status result "%s".'
% (rid, error_message))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' %
str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Delete YouTube: reportFailure ERROR',
'%s\n' % str(traceback.format_exception(et, ev, tb)))
class PyramidalClient (object):
"""Network client for generating pyramidal tiles.
"""
## Derived from the ermrest iobox service client
def __init__(self, **kwargs):
self.metadata = kwargs.get("metadata")
self.baseuri = kwargs.get("baseuri")
o = urlparse.urlparse(self.baseuri)
self.scheme = o[0]
host_port = o[1].split(":")
self.host = host_port[0]
self.path = o.path
self.port = None
if len(host_port) > 1:
self.port = host_port[1]
self.dzi = kwargs.get("dzi")
self.thumbnails = kwargs.get("thumbnails")
self.czi2dzi = kwargs.get("czi2dzi")
self.viewer = kwargs.get("viewer")
self.czirules = kwargs.get("czirules")
self.showinf = kwargs.get("showinf")
self.data_scratch = kwargs.get("data_scratch")
self.cookie = kwargs.get("cookie")
self.store = HatracStore(
self.scheme,
self.host,
{'cookie': self.cookie}
)
self.catalog = PollingErmrestCatalog(
self.scheme,
self.host,
self.path.split('/')[-1],
{'cookie': self.cookie}
)
self.mail_server = kwargs.get("mail_server")
self.mail_sender = kwargs.get("mail_sender")
self.mail_receiver = kwargs.get("mail_receiver")
self.logger = kwargs.get("logger")
self.logger.debug('Client initialized.')
"""
Send email notification
"""
def sendMail(self, subject, text):
if self.mail_server and self.mail_sender and self.mail_receiver:
retry = 0
ready = False
while not ready:
try:
msg = MIMEText('%s\n\n%s' % (text, mail_footer), 'plain')
msg['Subject'] = subject
msg['From'] = self.mail_sender
msg['To'] = self.mail_receiver
s = smtplib.SMTP(self.mail_server)
s.sendmail(self.mail_sender, self.mail_receiver.split(','), msg.as_string())
s.quit()
self.logger.debug('Sent email notification.')
ready = True
except socket.gaierror as e:
if e.errno == socket.EAI_AGAIN:
time.sleep(100)
retry = retry + 1
ready = retry > 10
else:
ready = True
if ready:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
ready = True
"""
Start the process for generating pyramidal tiles
"""
def start(self):
try:
self.processHistologicalImages()
except:
et, ev, tb = sys.exc_info()
self.logger.error('got unexpected exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Tiles: unexpected exception', '%s\nThe process might have been stopped\n' % str(traceback.format_exception(et, ev, tb)))
raise
def processHistologicalImages(self):
"""
Query for detecting new slides - the most recently first
"""
url = '/entity/Histological_Images:HE_Slide/!File_Bytes::null::&Pyramid_URL::null::/Processing_Status=in%%20progress;Processing_Status::null::@sort(%s::desc::)' % (urlquote('RCT'))
resp = self.catalog.get(url)
resp.raise_for_status()
slides = resp.json()
slideids = []
for slide in slides:
slideids.append((slide['ID'], slide['Filename'], slide['File_URL'], slide['RCT'], slide['File_MD5'], slide['Name'], slide['RID']))
self.logger.debug('Processing %d HistologicalImages slides(s).' % (len(slideids)))
for slideId,filename,file_url,creation_time,md5,name,rid in slideids:
self.logger.debug('Generating pyramidal tiles for the file "%s"' % (filename))
"""
Extract the file from hatrac
"""
f = self.getHatracFile(filename, file_url)
if f == None:
continue
"""
Create the directory for the tiles
"""
year = parse(creation_time).strftime("%Y")
outdir = '%s/%s/%s' % (self.dzi, year, md5)
if not os.path.exists(outdir):
os.makedirs(outdir)
"""
Convert the file to DZI
"""
returncode = self.convert2dzi(f, outdir)
if returncode != 0:
"""
Update the slide table with the failure result.
"""
self.updateAttributes('Histological_Images:HE_Slide',
rid,
["Thumbnail", "Processing_Status"],
{'RID': rid,
'Thumbnail': '/thumbnails/generic/generic_genetic.png',
'Processing_Status': 'czi2dzi error'
})
continue
"""
Generate the thumbnail
"""
thumbnail,urls = self.writeThumbnailImage(f, year, md5)
if thumbnail == None:
"""
Update the slide table with the failure result.
"""
self.updateAttributes('Histological_Images:HE_Slide',
rid,
["Thumbnail", "Processing_Status"],
{'RID': rid,
'Thumbnail': '/thumbnails/generic/generic_genetic.png',
'Processing_Status': 'DZI failure'
})
continue
"""
Extract the metadata
"""
self.logger.debug('Extracting metadata for filename "%s"' % (filename))
bioformatsClient = BioformatsClient(showinf=self.showinf, \
czirules=self.czirules, \
cziFile=f, \
logger=self.logger)
try:
metadata = bioformatsClient.getMetadata()
if metadata == None:
metadata = {}
self.logger.debug('Metadata: "%s"' % str(metadata))
os.remove('temp.xml')
except XMLSyntaxError:
et, ev, tb = sys.exc_info()
self.logger.error('got unexpected exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Tiles: XMLSyntaxError', '%s\n' % str(traceback.format_exception(et, ev, tb)))
metadata = {}
os.remove(f)
"""
Update the slide table with the success result.
"""
self.updateAttributes('Histological_Images:HE_Slide',
rid,
["Thumbnail","Pyramid_URL","Processing_Status","uri"],
{'RID': rid,
'Thumbnail': thumbnail,
'Pyramid_URL': '/%s?%s' % (self.viewer, urls),
'uri': '/%s?%s' % (self.viewer, urls),
"Processing_Status": 'success'
})
self.logger.debug('SUCCEEDED created the tiles directory for the file "%s".' % (filename))
"""
Update/Create the image entry with the metadata
"""
obj = {}
obj['ID'] = slideId
obj['Name'] = name
obj['url'] = '/chaise/viewer/#2/Histological_Images:HE_Slide/ID=%d' % slideId
columns = ['ID', 'Name', 'url']
for col in self.metadata:
if col in metadata and metadata[col] != None:
columns.append(col)
obj[col] = metadata[col]
"""
Check if we have an update or create
"""
rid = self.getRID('Histological_Images:HE_Image', 'ID=%d' % slideId)
if rid != None:
obj['RID'] = rid
self.updateAttributes('Histological_Images:HE_Image',
rid,
columns,
obj
)
else:
self.createEntity('Histological_Images:HE_Image', obj)
self.logger.debug('SUCCEEDED created the image entry for the file "%s".' % (filename))
self.logger.debug('Ended HistologicalImages Slides Processing.')
"""
Extract the file from hatrac
"""
def getHatracFile(self, filename, file_url):
try:
hatracFile = '%s/%s' % (self.data_scratch, filename)
self.store.get_obj(file_url, destfilename=hatracFile)
self.logger.debug('File "%s", %d bytes.' % (hatracFile, os.stat(hatracFile).st_size))
return hatracFile
except:
et, ev, tb = sys.exc_info()
self.logger.error('got unexpected exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Tiles: write thumbnail ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
return None
"""
Generate the thumbnail
"""
def writeThumbnailImage(self, filename, year, md5):
try:
scanDir='%s/%s/%s' % (self.dzi, year, md5)
channels = []
for channel in os.listdir(scanDir):
if os.path.isdir('%s%s%s' % (scanDir, os.sep, channel)):
channels.append( channel)
outdir = '%s/%s' % (self.thumbnails, year)
if not os.path.exists(outdir):
os.makedirs(outdir)
shutil.copyfile('%s/%s/%s/%s/0/0_0.jpg' % (self.dzi, year, md5, channels[0]), '%s/%s.jpg' % (outdir, md5))
thumbnail = '/thumbnails/%s/%s.jpg' % (urlquote(year), urlquote(md5))
urls = []
for channel in channels:
urls.append('url=/data/%s/%s/%s/ImageProperties.xml' % (year, md5, channel))
return (thumbnail, '&'.join(urls))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got unexpected exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Tiles: write thumbnail ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
os.remove(filename)
return (None, None)
"""
Convert the input file to DZI
"""
def convert2dzi(self, filename, outdir):
try:
currentDirectory=os.getcwd()
os.chdir(self.dzi)
args = [self.czi2dzi, filename, outdir]
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdoutdata, stderrdata = p.communicate()
returncode = p.returncode
os.chdir(currentDirectory)
if returncode != 0:
self.logger.error('Can not convert czi to dzi for file "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata))
self.sendMail('FAILURE Tiles', 'Can not convert czi to dzi for file "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata))
os.remove(filename)
except:
et, ev, tb = sys.exc_info()
self.logger.error('got unexpected exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Tiles: czi2dzi ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
os.chdir(currentDirectory)
self.logger.error('Can not generate pyramidal tiles for the file "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata))
self.sendMail('FAILURE Tiles', 'Can not generate pyramidal tiles for the file "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata))
returncode = 1
return returncode
"""
Update the ermrest attributes
"""
def updateAttributes (self, path, rid, columns, row):
"""
Update the ermrest attributes with the row values.
"""
try:
columns = ','.join([urlquote(col) for col in columns])
url = '/attributegroup/%s/RID;%s' % (path, columns)
resp = self.catalog.put(
url,
json=[row]
)
resp.raise_for_status()
self.logger.debug('SUCCEEDED updated the table "%s" for the RID "%s" with "%s".' % (path, rid, json.dumps(row, indent=4)))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Tiles: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
"""
Insert a row in a table
"""
def createEntity (self, path, row):
"""
Insert the row in the table.
"""
try:
url = '/entity/%s' % (path)
resp = self.catalog.post(
url,
json=[row]
)
resp.raise_for_status()
self.logger.debug('SUCCEEDED created in the table "%s" the entry "%s".' % (path, json.dumps(row, indent=4)))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Tiles: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
"""
Check if an entry exist in the table
Return the RID if it exists, and None otherwise
"""
def getRID (self, path, predicate):
"""
Get the RID of the row.
"""
try:
RID = None
url = '/entity/%s/%s' % (path, predicate)
resp = self.catalog.get(url)
resp.raise_for_status()
rows = resp.json()
if len(rows) == 1:
RID = rows[0]['RID']
self.logger.debug('RID for the url = "%s" is "%s".' % (url, RID))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Tiles: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
return RID
class YouTubeClient (object):
"""
Network client for YouTube.
"""
## Derived from the ermrest iobox service client
def __init__(self, **kwargs):
self.baseuri = kwargs.get("baseuri")
o = urlparse.urlparse(self.baseuri)
self.scheme = o[0]
host_port = o[1].split(":")
self.host = host_port[0]
self.path = o.path
self.port = None
if len(host_port) > 1:
self.port = host_port[1]
self.cookie = kwargs.get("cookie")
self.client_secrets_file = kwargs.get("client_secrets_file")
self.client_oauth2_file = kwargs.get("client_oauth2_file")
self.catalog = PollingErmrestCatalog(
self.scheme,
self.host,
self.path.split('/')[-1],
{'cookie': self.cookie}
)
self.mail_server = kwargs.get("mail_server")
self.mail_sender = kwargs.get("mail_sender")
self.mail_receiver = kwargs.get("mail_receiver")
self.logger = kwargs.get("logger")
self.logger.debug('Delete YouTube Client initialized.')
"""
Send email notification
"""
def sendMail(self, subject, text):
if self.mail_server and self.mail_sender and self.mail_receiver:
retry = 0
ready = False
while not ready:
try:
msg = MIMEText('%s\n\n%s' % (text, mail_footer), 'plain')
msg['Subject'] = subject
msg['From'] = self.mail_sender
msg['To'] = self.mail_receiver
s = smtplib.SMTP(self.mail_server)
s.sendmail(self.mail_sender, self.mail_receiver.split(','), msg.as_string())
s.quit()
self.logger.debug('Sent email notification.')
ready = True
except socket.gaierror as e:
if e.errno == socket.EAI_AGAIN:
time.sleep(100)
retry = retry + 1
ready = retry > 10
else:
ready = True
if ready:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
ready = True
"""
Start the process for deleting files from YouTube
"""
def start(self):
try:
self.deleteFromYouTube()
except:
et, ev, tb = sys.exc_info()
self.logger.error('got unexpected exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Delete YouTube: unexpected exception', '%s\nThe process might have been stopped\n' % str(traceback.format_exception(et, ev, tb)))
raise
"""
Get the YouTube Delete credentials
"""
def youtube_authenticated_service(self):
flow = InstalledAppFlow.from_client_secrets_file(self.client_secrets_file, SCOPES)
storage = Storage(self.client_oauth2_file)
credentials = storage.get()
if credentials is None or credentials.invalid:
credentials = flow.run_console()
self.youtube = build(API_SERVICE_NAME, API_VERSION, credentials = credentials)
"""
Delete a video from YouTube
"""
def youtube_delete(self, youtube_uri):
res = False
try:
id = youtube_uri[youtube_uri.rfind('/')+1:youtube_uri.index('?')]
self.logger.debug('Deleting YouTube video id="%s".' % (id))
self.youtube_authenticated_service()
if self.youtube is not None:
self.logger.debug('Authenticated to the YouTube delete service.')
response = self.youtube.videos().delete(id=id).execute()
self.logger.debug('Deleted response %s.' % (response))
res = True
else:
self.logger.debug('Authentication for deleting a YouTube video failed.')
except:
et, ev, tb = sys.exc_info()
self.logger.error('got YouTube exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
return res
"""
Delete videos from YouTube
"""
def deleteFromYouTube(self):
url = '/entity/Common:Delete_Youtube/Youtube_Deleted=FALSE/Processing_Status=in%20progress;Processing_Status::null::'
resp = self.catalog.get(url)
resp.raise_for_status()
files = resp.json()
fileids = []
for f in files:
fileids.append((f['YouTube_URI'], f['RID']))
self.logger.debug('Deleting from YouTube %d videos(s).' % (len(fileids)))
for youtube_uri,rid in fileids:
try:
youtube_deleted = self.youtube_delete(youtube_uri)
if youtube_deleted == True:
self.logger.debug('SUCCEEDED deleted from YouTube the video with the URL: "%s".' % (youtube_uri))
columns = ["Youtube_Deleted", "Processing_Status"]
columns = ','.join([urlquote(col) for col in columns])
url = '/attributegroup/Common:Delete_Youtube/RID;%s' % (columns)
obj = {'RID': rid,
'Youtube_Deleted': True,
'Processing_Status': 'success'
}
self.catalog.put(
url,
json=[obj]
)
self.logger.debug('SUCCEEDED updated the Common:Delete_Youtube table entry for the YouTube URL: "%s".' % (youtube_uri))
else:
self.logger.debug('Failure in deleting from YouTube the video with the URL: "%s".' % (youtube_uri))
self.sendMail('FAILURE Delete YouTube: YouTube Failure', 'The video "%s" could not be deleted from Youtube.' % youtube_uri)
self.reportFailure(rid, 'YouTube Failure')
except Exception as e:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
self.reportFailure(rid, str(e))
"""
Update the Delete_Youtube table with the ERROR status
"""
def reportFailure(self, rid, error_message):
"""
Update the Delete_Youtube table with the failure result.
"""
try:
columns = ["Processing_Status"]
columns = ','.join([urlquote(col) for col in columns])
url = '/attributegroup/Common:Delete_Youtube/RID;%s' % (columns)
obj = {'RID': rid,
'Processing_Status': '%s' % error_message
}
self.catalog.put(
url,
json=[obj]
)
self.logger.debug('SUCCEEDED updated the Delete_Youtube table for the RID "%s" with the Processing_Status result "%s".' % (rid, error_message))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Delete YouTube: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
class PyramidalClient(object):
"""Network client for generating pyramidal tiles.
"""
## Derived from the ermrest iobox service client
def __init__(self, **kwargs):
self.metadata = kwargs.get("metadata")
self.baseuri = kwargs.get("baseuri")
o = urlparse.urlparse(self.baseuri)
self.scheme = o[0]
host_port = o[1].split(":")
self.host = host_port[0]
self.path = o.path
self.port = None
if len(host_port) > 1:
self.port = host_port[1]
self.dzi = kwargs.get("dzi")
self.thumbnails = kwargs.get("thumbnails")
self.czi2dzi = kwargs.get("czi2dzi")
self.viewer = kwargs.get("viewer")
self.czirules = kwargs.get("czirules")
self.showinf = kwargs.get("showinf")
self.data_scratch = kwargs.get("data_scratch")
self.cookie = kwargs.get("cookie")
self.store = HatracStore(self.scheme, self.host,
{'cookie': self.cookie})
self.catalog = PollingErmrestCatalog(self.scheme, self.host,
self.path.split('/')[-1],
{'cookie': self.cookie})
self.mail_server = kwargs.get("mail_server")
self.mail_sender = kwargs.get("mail_sender")
self.mail_receiver = kwargs.get("mail_receiver")
self.logger = kwargs.get("logger")
self.logger.debug('Client initialized.')
"""
Send email notification
"""
def sendMail(self, subject, text):
if self.mail_server and self.mail_sender and self.mail_receiver:
retry = 0
ready = False
while not ready:
try:
msg = MIMEText('%s\n\n%s' % (text, mail_footer), 'plain')
msg['Subject'] = subject
msg['From'] = self.mail_sender
msg['To'] = self.mail_receiver
s = smtplib.SMTP(self.mail_server)
s.sendmail(self.mail_sender, self.mail_receiver.split(','),
msg.as_string())
s.quit()
self.logger.debug('Sent email notification.')
ready = True
except socket.gaierror as e:
if e.errno == socket.EAI_AGAIN:
time.sleep(100)
retry = retry + 1
ready = retry > 10
else:
ready = True
if ready:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error(
'%s' % str(traceback.format_exception(et, ev, tb)))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error(
'%s' % str(traceback.format_exception(et, ev, tb)))
ready = True
"""
Start the process for generating pyramidal tiles
"""
def start(self):
try:
self.processHistologicalImages()
except:
et, ev, tb = sys.exc_info()
self.logger.error('got unexpected exception "%s"' % str(ev))
self.logger.error('%s' %
str(traceback.format_exception(et, ev, tb)))
self.sendMail(
'FAILURE Tiles: unexpected exception',
'%s\nThe process might have been stopped\n' %
str(traceback.format_exception(et, ev, tb)))
raise
def processHistologicalImages(self):
"""
Query for detecting new slides - the most recently first
"""
url = '/entity/Histological_Images:HE_Slide/!File_Bytes::null::&Pyramid_URL::null::/Processing_Status=in%%20progress;Processing_Status::null::@sort(%s::desc::)' % (
urlquote('RCT'))
resp = self.catalog.get(url)
resp.raise_for_status()
slides = resp.json()
slideids = []
for slide in slides:
slideids.append(
(slide['ID'], slide['Filename'], slide['File_URL'],
slide['RCT'], slide['File_MD5'], slide['Name'], slide['RID']))
self.logger.debug('Processing %d HistologicalImages slides(s).' %
(len(slideids)))
for slideId, filename, file_url, creation_time, md5, name, rid in slideids:
self.logger.debug('Generating pyramidal tiles for the file "%s"' %
(filename))
"""
Extract the file from hatrac
"""
f = self.getHatracFile(filename, file_url)
if f == None:
continue
"""
Create the directory for the tiles
"""
year = parse(creation_time).strftime("%Y")
outdir = '%s/%s/%s' % (self.dzi, year, md5)
if not os.path.exists(outdir):
os.makedirs(outdir)
"""
Convert the file to DZI
"""
returncode = self.convert2dzi(f, outdir)
if returncode != 0:
"""
Update the slide table with the failure result.
"""
self.updateAttributes(
'Histological_Images:HE_Slide', rid,
["Thumbnail", "Processing_Status"], {
'RID': rid,
'Thumbnail': '/thumbnails/generic/generic_genetic.png',
'Processing_Status': 'czi2dzi error'
})
continue
"""
Generate the thumbnail
"""
thumbnail, urls = self.writeThumbnailImage(f, year, md5)
if thumbnail == None:
"""
Update the slide table with the failure result.
"""
self.updateAttributes(
'Histological_Images:HE_Slide', rid,
["Thumbnail", "Processing_Status"], {
'RID': rid,
'Thumbnail': '/thumbnails/generic/generic_genetic.png',
'Processing_Status': 'DZI failure'
})
continue
"""
Extract the metadata
"""
self.logger.debug('Extracting metadata for filename "%s"' %
(filename))
bioformatsClient = BioformatsClient(showinf=self.showinf, \
czirules=self.czirules, \
cziFile=f, \
logger=self.logger)
try:
metadata = bioformatsClient.getMetadata()
if metadata == None:
metadata = {}
self.logger.debug('Metadata: "%s"' % str(metadata))
os.remove('temp.xml')
except XMLSyntaxError:
et, ev, tb = sys.exc_info()
self.logger.error('got unexpected exception "%s"' % str(ev))
self.logger.error('%s' %
str(traceback.format_exception(et, ev, tb)))
self.sendMail(
'FAILURE Tiles: XMLSyntaxError',
'%s\n' % str(traceback.format_exception(et, ev, tb)))
metadata = {}
os.remove(f)
"""
Update the slide table with the success result.
"""
self.updateAttributes(
'Histological_Images:HE_Slide', rid,
["Thumbnail", "Pyramid_URL", "Processing_Status", "uri"], {
'RID': rid,
'Thumbnail': thumbnail,
'Pyramid_URL': '/%s?%s' % (self.viewer, urls),
'uri': '/%s?%s' % (self.viewer, urls),
"Processing_Status": 'success'
})
self.logger.debug(
'SUCCEEDED created the tiles directory for the file "%s".' %
(filename))
"""
Update/Create the image entry with the metadata
"""
obj = {}
obj['ID'] = slideId
obj['Name'] = name
obj['url'] = '/chaise/viewer/#2/Histological_Images:HE_Slide/ID=%d' % slideId
columns = ['ID', 'Name', 'url']
for col in self.metadata:
if col in metadata and metadata[col] != None:
columns.append(col)
obj[col] = metadata[col]
"""
Check if we have an update or create
"""
rid = self.getRID('Histological_Images:HE_Image',
'ID=%d' % slideId)
if rid != None:
obj['RID'] = rid
self.updateAttributes('Histological_Images:HE_Image', rid,
columns, obj)
else:
self.createEntity('Histological_Images:HE_Image', obj)
self.logger.debug(
'SUCCEEDED created the image entry for the file "%s".' %
(filename))
self.logger.debug('Ended HistologicalImages Slides Processing.')
"""
Extract the file from hatrac
"""
def getHatracFile(self, filename, file_url):
try:
hatracFile = '%s/%s' % (self.data_scratch, filename)
self.store.get_obj(file_url, destfilename=hatracFile)
self.logger.debug('File "%s", %d bytes.' %
(hatracFile, os.stat(hatracFile).st_size))
return hatracFile
except:
et, ev, tb = sys.exc_info()
self.logger.error('got unexpected exception "%s"' % str(ev))
self.logger.error('%s' %
str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Tiles: write thumbnail ERROR',
'%s\n' % str(traceback.format_exception(et, ev, tb)))
return None
"""
Generate the thumbnail
"""
def writeThumbnailImage(self, filename, year, md5):
try:
scanDir = '%s/%s/%s' % (self.dzi, year, md5)
channels = []
for channel in os.listdir(scanDir):
if os.path.isdir('%s%s%s' % (scanDir, os.sep, channel)):
channels.append(channel)
outdir = '%s/%s' % (self.thumbnails, year)
if not os.path.exists(outdir):
os.makedirs(outdir)
shutil.copyfile(
'%s/%s/%s/%s/0/0_0.jpg' % (self.dzi, year, md5, channels[0]),
'%s/%s.jpg' % (outdir, md5))
thumbnail = '/thumbnails/%s/%s.jpg' % (urlquote(year),
urlquote(md5))
urls = []
for channel in channels:
urls.append('url=/data/%s/%s/%s/ImageProperties.xml' %
(year, md5, channel))
return (thumbnail, '&'.join(urls))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got unexpected exception "%s"' % str(ev))
self.logger.error('%s' %
str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Tiles: write thumbnail ERROR',
'%s\n' % str(traceback.format_exception(et, ev, tb)))
os.remove(filename)
return (None, None)
"""
Convert the input file to DZI
"""
def convert2dzi(self, filename, outdir):
try:
currentDirectory = os.getcwd()
os.chdir(self.dzi)
args = [self.czi2dzi, filename, outdir]
p = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdoutdata, stderrdata = p.communicate()
returncode = p.returncode
os.chdir(currentDirectory)
if returncode != 0:
self.logger.error(
'Can not convert czi to dzi for file "%s".\nstdoutdata: %s\nstderrdata: %s\n'
% (filename, stdoutdata, stderrdata))
self.sendMail(
'FAILURE Tiles',
'Can not convert czi to dzi for file "%s".\nstdoutdata: %s\nstderrdata: %s\n'
% (filename, stdoutdata, stderrdata))
os.remove(filename)
except:
et, ev, tb = sys.exc_info()
self.logger.error('got unexpected exception "%s"' % str(ev))
self.logger.error('%s' %
str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Tiles: czi2dzi ERROR',
'%s\n' % str(traceback.format_exception(et, ev, tb)))
os.chdir(currentDirectory)
self.logger.error(
'Can not generate pyramidal tiles for the file "%s".\nstdoutdata: %s\nstderrdata: %s\n'
% (filename, stdoutdata, stderrdata))
self.sendMail(
'FAILURE Tiles',
'Can not generate pyramidal tiles for the file "%s".\nstdoutdata: %s\nstderrdata: %s\n'
% (filename, stdoutdata, stderrdata))
returncode = 1
return returncode
"""
Update the ermrest attributes
"""
def updateAttributes(self, path, rid, columns, row):
"""
Update the ermrest attributes with the row values.
"""
try:
columns = ','.join([urlquote(col) for col in columns])
url = '/attributegroup/%s/RID;%s' % (path, columns)
resp = self.catalog.put(url, json=[row])
resp.raise_for_status()
self.logger.debug(
'SUCCEEDED updated the table "%s" for the RID "%s" with "%s".'
% (path, rid, json.dumps(row, indent=4)))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' %
str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Tiles: reportFailure ERROR',
'%s\n' % str(traceback.format_exception(et, ev, tb)))
"""
Insert a row in a table
"""
def createEntity(self, path, row):
"""
Insert the row in the table.
"""
try:
url = '/entity/%s' % (path)
resp = self.catalog.post(url, json=[row])
resp.raise_for_status()
self.logger.debug(
'SUCCEEDED created in the table "%s" the entry "%s".' %
(path, json.dumps(row, indent=4)))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' %
str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Tiles: reportFailure ERROR',
'%s\n' % str(traceback.format_exception(et, ev, tb)))
"""
Check if an entry exist in the table
Return the RID if it exists, and None otherwise
"""
def getRID(self, path, predicate):
"""
Get the RID of the row.
"""
try:
RID = None
url = '/entity/%s/%s' % (path, predicate)
resp = self.catalog.get(url)
resp.raise_for_status()
rows = resp.json()
if len(rows) == 1:
RID = rows[0]['RID']
self.logger.debug('RID for the url = "%s" is "%s".' % (url, RID))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' %
str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Tiles: reportFailure ERROR',
'%s\n' % str(traceback.format_exception(et, ev, tb)))
return RID
class HatracClient (object):
"""
Network client for hatrac.
"""
## Derived from the ermrest iobox service client
def __init__(self, **kwargs):
self.baseuri = kwargs.get("baseuri")
o = urlparse.urlparse(self.baseuri)
self.scheme = o[0]
host_port = o[1].split(":")
self.host = host_port[0]
self.path = o.path
self.port = None
if len(host_port) > 1:
self.port = host_port[1]
self.cookie = kwargs.get("cookie")
self.store = HatracStore(
self.scheme,
self.host,
{'cookie': self.cookie}
)
self.catalog = PollingErmrestCatalog(
self.scheme,
self.host,
self.path.split('/')[-1],
{'cookie': self.cookie}
)
self.mail_server = kwargs.get("mail_server")
self.mail_sender = kwargs.get("mail_sender")
self.mail_receiver = kwargs.get("mail_receiver")
self.logger = kwargs.get("logger")
self.logger.debug('Hatrac Client initialized.')
"""
Send email notification
"""
def sendMail(self, subject, text):
if self.mail_server and self.mail_sender and self.mail_receiver:
retry = 0
ready = False
while not ready:
try:
msg = MIMEText('%s\n\n%s' % (text, mail_footer), 'plain')
msg['Subject'] = subject
msg['From'] = self.mail_sender
msg['To'] = self.mail_receiver
s = smtplib.SMTP(self.mail_server)
s.sendmail(self.mail_sender, self.mail_receiver.split(','), msg.as_string())
s.quit()
self.logger.debug('Sent email notification.')
ready = True
except socket.gaierror as e:
if e.errno == socket.EAI_AGAIN:
time.sleep(100)
retry = retry + 1
ready = retry > 10
else:
ready = True
if ready:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
ready = True
"""
Start the process for deleting files from hatrac
"""
def start(self):
try:
self.deleteFromHatrac()
except:
et, ev, tb = sys.exc_info()
self.logger.error('got unexpected exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Delete Hatrac: unexpected exception', '%s\nThe process might have been stopped\n' % str(traceback.format_exception(et, ev, tb)))
raise
"""
Delete videos from hatrac
"""
def deleteFromHatrac(self):
url = '/entity/Common:Delete_Hatrac/Hatrac_Deleted=FALSE/Processing_Status=in%20progress;Processing_Status::null::'
resp = self.catalog.get(url)
resp.raise_for_status()
files = resp.json()
fileids = []
for f in files:
fileids.append((f['Hatrac_URI'], f['RID']))
self.logger.debug('Deleting from hatrac %d files(s).' % (len(fileids)))
for hatrac_uri,rid in fileids:
try:
self.store.del_obj(hatrac_uri)
self.logger.debug('SUCCEEDED deleted from hatrac the "%s" file.' % (hatrac_uri))
columns = ["Hatrac_Deleted", "Processing_Status"]
columns = ','.join([urlquote(col) for col in columns])
url = '/attributegroup/Common:Delete_Hatrac/RID;%s' % (columns)
obj = {'RID': rid,
'Hatrac_Deleted': True,
'Processing_Status': 'success'
}
self.catalog.put(
url,
json=[obj]
)
self.logger.debug('SUCCEEDED updated the Common:Delete_Hatrac table entry for the Hatrac URL: "%s".' % (hatrac_uri))
except Exception as e:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
self.reportFailure(rid, str(e))
"""
Update the Delete_Hatrac table with the ERROR status
"""
def reportFailure(self, rid, error_message):
"""
Update the Delete_Hatrac table with the failure result.
"""
try:
columns = ["Processing_Status"]
columns = ','.join([urlquote(col) for col in columns])
url = '/attributegroup/Common:Delete_Hatrac/RID;%s' % (columns)
obj = {'RID': rid,
'Processing_Status': '%s' % error_message
}
self.catalog.put(
url,
json=[obj]
)
self.logger.debug('SUCCEEDED updated the Delete_Hatrac table for the RID "%s" with the Processing_Status result "%s".' % (rid, error_message))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Delete Hatrac: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
class HatracClient(object):
"""
Network client for hatrac.
"""
## Derived from the ermrest iobox service client
def __init__(self, **kwargs):
self.baseuri = kwargs.get("baseuri")
o = urlparse.urlparse(self.baseuri)
self.scheme = o[0]
host_port = o[1].split(":")
self.host = host_port[0]
self.path = o.path
self.port = None
if len(host_port) > 1:
self.port = host_port[1]
self.cookie = kwargs.get("cookie")
self.store = HatracStore(self.scheme, self.host,
{'cookie': self.cookie})
self.catalog = PollingErmrestCatalog(self.scheme, self.host,
self.path.split('/')[-1],
{'cookie': self.cookie})
self.mail_server = kwargs.get("mail_server")
self.mail_sender = kwargs.get("mail_sender")
self.mail_receiver = kwargs.get("mail_receiver")
self.logger = kwargs.get("logger")
self.logger.debug('Hatrac Client initialized.')
"""
Send email notification
"""
def sendMail(self, subject, text):
if self.mail_server and self.mail_sender and self.mail_receiver:
retry = 0
ready = False
while not ready:
try:
msg = MIMEText('%s\n\n%s' % (text, mail_footer), 'plain')
msg['Subject'] = subject
msg['From'] = self.mail_sender
msg['To'] = self.mail_receiver
s = smtplib.SMTP(self.mail_server)
s.sendmail(self.mail_sender, self.mail_receiver.split(','),
msg.as_string())
s.quit()
self.logger.debug('Sent email notification.')
ready = True
except socket.gaierror as e:
if e.errno == socket.EAI_AGAIN:
time.sleep(100)
retry = retry + 1
ready = retry > 10
else:
ready = True
if ready:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error(
'%s' % str(traceback.format_exception(et, ev, tb)))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error(
'%s' % str(traceback.format_exception(et, ev, tb)))
ready = True
"""
Start the process for deleting files from hatrac
"""
def start(self):
try:
self.deleteFromHatrac()
except:
et, ev, tb = sys.exc_info()
self.logger.error('got unexpected exception "%s"' % str(ev))
self.logger.error('%s' %
str(traceback.format_exception(et, ev, tb)))
self.sendMail(
'FAILURE Delete Hatrac: unexpected exception',
'%s\nThe process might have been stopped\n' %
str(traceback.format_exception(et, ev, tb)))
raise
"""
Delete videos from hatrac
"""
def deleteFromHatrac(self):
url = '/entity/Common:Delete_Hatrac/Hatrac_Deleted=FALSE/Processing_Status=in%20progress;Processing_Status::null::'
resp = self.catalog.get(url)
resp.raise_for_status()
files = resp.json()
fileids = []
for f in files:
fileids.append((f['Hatrac_URI'], f['RID']))
self.logger.debug('Deleting from hatrac %d files(s).' % (len(fileids)))
for hatrac_uri, rid in fileids:
try:
self.store.del_obj(hatrac_uri)
self.logger.debug(
'SUCCEEDED deleted from hatrac the "%s" file.' %
(hatrac_uri))
columns = ["Hatrac_Deleted", "Processing_Status"]
columns = ','.join([urlquote(col) for col in columns])
url = '/attributegroup/Common:Delete_Hatrac/RID;%s' % (columns)
obj = {
'RID': rid,
'Hatrac_Deleted': True,
'Processing_Status': 'success'
}
self.catalog.put(url, json=[obj])
self.logger.debug(
'SUCCEEDED updated the Common:Delete_Hatrac table entry for the Hatrac URL: "%s".'
% (hatrac_uri))
except Exception as e:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' %
str(traceback.format_exception(et, ev, tb)))
self.reportFailure(rid, str(e))
"""
Update the Delete_Hatrac table with the ERROR status
"""
def reportFailure(self, rid, error_message):
"""
Update the Delete_Hatrac table with the failure result.
"""
try:
columns = ["Processing_Status"]
columns = ','.join([urlquote(col) for col in columns])
url = '/attributegroup/Common:Delete_Hatrac/RID;%s' % (columns)
obj = {'RID': rid, 'Processing_Status': '%s' % error_message}
self.catalog.put(url, json=[obj])
self.logger.debug(
'SUCCEEDED updated the Delete_Hatrac table for the RID "%s" with the Processing_Status result "%s".'
% (rid, error_message))
except:
et, ev, tb = sys.exc_info()
self.logger.error('got exception "%s"' % str(ev))
self.logger.error('%s' %
str(traceback.format_exception(et, ev, tb)))
self.sendMail('FAILURE Delete Hatrac: reportFailure ERROR',
'%s\n' % str(traceback.format_exception(et, ev, tb)))