# debug with `DEBUG=leicacam python script.py`
if platform.system() == 'Windows':
# monkeypatch
@logger
def debug(msg):
"""Debug on Windows."""
try:
dbg = os.environ['DEBUG']
if dbg == 'leicacam' or dbg == '*':
print('leicacam ' + str(msg))
except KeyError:
pass
else:
debug = logger(pydebug.debug('leicacam')) # pylint: disable=invalid-name
class CAM(object):
"""Driver for LASAF Computer Assisted Microscopy."""
# pylint: disable=too-many-instance-attributes
def __init__(self, host='127.0.0.1', port=8895):
"""Set up instance."""
self.host = host
self.port = port
# prefix for all commands
self.prefix = [('cli', 'python-leicacam'), ('app', 'matrix')]
self.prefix_bytes = b'/cli:python-leicacam /app:matrix '
self.buffer_size = 1024
from scrappy.persistor.document import Document
from scrappy.core.crash_dump import crash_dump
from scrappy.util.tmpFile import tmpFile
from scrappy.core.utils import ensure_dir
from scrappy.core.commands import Die
from os import path, makedirs
from threading import Thread
from queue import Queue
from uuid import uuid4
import pydebug
debug = pydebug.debug("persistor")
class Persistor(Thread):
"""Simple persistors that saves to files on disk
"""
def __init__(self, base_path):
Thread.__init__(self)
self._id = uuid4()
self.queue = Queue()
self.base_path = base_path
def save_one_sync(self, document):
"""Saves one document
Arguments:
document {Document} -- Document injected into this function
Raises:
import pydebug
import time
debug = pydebug.debug("test")
debug("hi there")
debug("hi there2")
debug2 = pydebug.debug("test2")
debug2("awesome")
debug2("even more awesome")
print 1, 2, 3, 4
debug(1, 2, 3, 4)
debug({"test": 1})
debug({"awesome": True}, True, 1, "test")
debug([], {1, 2, 3}, {"awesome": True}, True, 1, "test")
time.sleep(0.1)
debug('should be milliseconds now')
time.sleep(1)
debug('should be seconds now')
debug(dir([]), repr([]), str([]))
from scrappy.driver.worker import Worker
from scrappy.driver.pool import create, dispose
from scrappy.tasks.task import Task
from itertools import cycle
from threading import Thread
from queue import Queue
import pydebug
debug = pydebug.debug("scheduler")
class Scheduler(Thread):
def __init__(self, pool_size, headless=True):
Thread.__init__(self)
self.pool = list()
self.retry = Queue()
self.tasks = Queue()
self.pool_size = pool_size
self.headless = headless
self.pool = create(pool_size, headless)
def __enter__(self):
return self
def __exit__(self, *arg):
self.start()
dispose(self.pool)
def schedule_task(self, task):
"""Enqueues a task
"""
from joblib import Parallel, delayed
from multiprocessing import cpu_count
try:
_pools = cpu_count()
except NotImplementedError:
_pools = 4
# compress
import json
from PIL import Image
from PIL.ImagePalette import ImagePalette
from copy import copy
# debug with `DEBUG=leicaexperiment python script.py`
debug = pydebug.debug('leicaexperiment')
# glob for consistent cross platform behavior
def glob(pattern):
"Sorted glob."
from glob import glob as sysglob
return sorted(sysglob(pattern))
# variables in case custom folders
_slide = 'slide'
_chamber = 'chamber'
_field = 'field'
_image = 'image'
_additional_data = 'AdditionalData'
_scanning_template = r'{ScanningTemplate}'
#!/usr/bin/env python3
# pylint: disable=anomalous-backslash-in-string
import pydebug
import sys
import svgwrite
import json
import math
from hurry.filesize import size
debug = pydebug.debug("svg")
status_map = {
"+": "finished",
"?": "non-tried",
"*": "non-trimmed",
"/": "non-scraped",
"-": "bad sector",
}
fill_map = {
"+": "green",
"?": "lightgray",
"*": "darkgray",
"/": "yellow",
"-": "red",
}
CSS_STYLES = """
.func_g:hover { stroke:black; stroke-width:0.5; cursor:pointer; }
"""
from scrappy.tasks.task import Task
from scrappy.persistor.in_memory import InMemoryPersistor
from scrappy.scheduler.round_robin import RoundRobin
from scrappy.persistor.document import Document
import json
import code
import re
import pydebug
debug = pydebug.debug("driver:task")
class LocaisRealizacao(Task):
def __init__(self, persistor):
Task.__init__(self, persistor)
def bootstrap(self, driver):
driver.get('https://sigaa.ufpb.br/sigaa/logon.jsf')
box = driver.find_element_by_css_selector('#form\:login')
box.clear()
box.click()
box.send_keys("lincolneloi")
box = driver.find_element_by_css_selector('#form\:senha')
box.click()
box.send_keys("lin987")
driver.find_element_by_css_selector('#form\:entrar').click()
from contextlib import contextmanager
from scrappy.core import crash_dump
from queue import Empty, Queue
from selenium import webdriver
from threading import Thread, Event
from uuid import uuid4
from scrappy.core.commands import Die
import time
import pydebug
debug = pydebug.debug("driver:worker")
class Worker(Thread):
"""Scrapper worker
A Worker continuously consumes and executes tasks from its internal queue
until a Die task arrives.
"""
def __init__(self, headless=True):
Thread.__init__(self)
self.driver = None
self.headless = headless
self._id = uuid4()
self.queue = Queue()
self.is_open = False
def __del__(self):
self.dispose()
def __enter__(self):
return self
from scrappy.persistor.persistor import Persistor
import pydebug
debug = pydebug.debug("persistor:dummy")
class DummyPersistor():
def save_one(self, document):
debug(document.data)
from time import sleep, time
from collections import OrderedDict
import socket, pydebug, platform, os
# debug with `DEBUG=leicacam python script.py`
if platform.system() == 'Windows':
# monkeypatch
def debug(msg):
try:
dbg = os.environ['DEBUG']
if dbg == 'leicacam' or dbg == '*':
print('leicacam ' + str(msg))
except KeyError:
pass
else:
debug = pydebug.debug('leicacam')
class CAM:
"Driver for LASAF Computer Assisted Microscopy."
def __init__(self, host='127.0.0.1', port=8895):
self.host = host
self.port = port
# prefix for all commands
self.prefix = [('cli', 'python-leicacam'),
('app', 'matrix')]
self.prefix_bytes = b'/cli:python-leicacam /app:matrix '
self.buffer_size = 1024
self.delay = 0.1 # poll every 100ms when waiting for incomming
self.connect()
"""Control microscope through LASAF Computer Assisted Microscopy."""
import select
import socket
from collections import OrderedDict
from time import sleep, time
import pydebug
# debug with `DEBUG=matrixscreener python script.py`
DEBUG = pydebug.debug('matrixscreener')
class CAM(object):
"""Driver for LASAF Computer Assisted Microscopy."""
def __init__(self, host='127.0.0.1', port=8895):
"""Set up instance."""
self.host = host
self.port = port
# prefix for all commands
self.prefix = [('cli', 'python-matrixscreener'),
('app', 'matrix')]
self.prefix_bytes = b'/cli:python-matrixscreener /app:matrix '
self.buffer_size = 1024
self.delay = 5e-2 # wait 50ms after sending commands
self.timeout = 10.0
self.connect()
def connect(self):
"""Connect to LASAF through a CAM-socket."""
self.socket = socket.socket()
# encoding: utf-8
"""
Run Fiji is just ImageJ macros headless with python.
"""
import pydebug, subprocess, os, re
from tempfile import mkstemp
import fijibin
# debug with DEBUG=fijibin python script.py
debug = pydebug.debug('fijibin')
##
# Running macros
##
def run(macro, output_files=[], force_close=True):
"""
Runs Fiji with the suplied macro. Output of Fiji can be viewed by
setting environment variable `DEBUG=fijibin`.
Parameters
----------
macro : string or list of strings
IJM-macro(s) to run. If list of strings, it will be joined with
a space, so all statements should end with ``;``.
output_files : list
Files to check if exists after macro has been run. Files specified that
do not exist after macro is done will print a warning message.
force_close : bool
Will add ``eval("script", "System.exit(42);");`` to end of macro. Exit
code 42 is used to overcome that errors in macro efficiently will exit
from time import gmtime, strftime
from scrappy.util.tmpFile import tmpFile
from uuid import uuid4
import pydebug
debug = pydebug.debug("core:logger")
def format_error(error):
"""
format_error(error) => (string, integer)
Formats an error message, returning formated output and generated id
Keyword arguments:
"""
time = strftime("%Y-%m-%d-%H:%M:%S", gmtime())
uuid = uuid4()
error = '---\nid: {id}\ntime: {time}\nmessage: "{msg}"\n---'.format(
id=uuid, time=time, msg=str(error))
return (error, uuid)
def crash_dump(caller_id, error, path=tmpFile("scrappy.dump")):
"""
Securely logs an error
Keyword arguments:
# encoding: utf-8
"""
Run Fiji is just ImageJ macros headless with python.
"""
import pydebug, subprocess, os, re
from tempfile import mkstemp
import fijibin
# debug with DEBUG=fijibin python script.py
debug = pydebug.debug('fijibin')
##
# Running macros
##
def run(macro, output_files=[], force_close=True):
"""
Runs Fiji with the suplied macro. Output of Fiji can be viewed by
setting environment variable `DEBUG=fijibin`.
Parameters
----------
macro : string or list of strings
IJM-macro(s) to run. If list of strings, it will be joined with
a space, so all statements should end with ``;``.
output_files : list
Files to check if exists after macro has been run. Files specified that
do not exist after macro is done will print a warning message.
force_close : bool
Will add ``eval("script", "System.exit(42);");`` to end of macro. Exit
code 42 is used to overcome that errors in macro efficiently will exit
from scrappy import app
from scrappy.core.crash_dump import crash_dump
from flask import jsonify, request
import pydebug
debug = pydebug.debug("distributed:crash_dump")
@app.route("/crash_dump", methods=['POST'])
def crash_dump_route():
data = request.get_json(force=True)
try:
crash_dump(data['caller_id'], data['error'])
return '', 200
except Exception as error:
debug("Error while trying to write crash dump: {}".format(str(error)))
return '', 400
import pydebug
import time
debug = pydebug.debug("test")
debug("hi there")
debug("hi there2")
debug2 = pydebug.debug("test2")
debug2("awesome")
debug2("even more awesome")
print 1, 2, 3, 4
debug(1, 2, 3, 4)
debug({"test": 1})
debug({"awesome": True}, True, 1, "test")
debug([], {1, 2, 3}, {"awesome": True}, True, 1, "test")
time.sleep(0.1)
debug('should be milliseconds now')
time.sleep(1)
debug('should be seconds now')
debug(dir([]), repr([]), str([]))
from time import sleep
from collections import OrderedDict
import socket, pydebug
# debug with `DEBUG=matrixscreener python script.py`
debug = pydebug.debug('matrixscreener')
class CAM:
"Driver for LASAF Computer Assisted Microscopy."
def __init__(self, host='127.0.0.1', port=8895):
self.host = host
self.port = port
# prefix for all commands
self.prefix = [('cli', 'python-matrixscreener'),
('app', 'matrix')]
self.prefix_bytes = b'/cli:python-matrixscreener /app:matrix '
self.buffer_size = 1024
self.delay = 5e-2 # wait 50ms after sending commands
self.connect()
def connect(self):
"Connects to LASAF through a CAM-socket."
self.socket = socket.socket()
self.socket.connect((self.host, self.port))
self.socket.settimeout(False) # non-blocking
sleep(self.delay) # wait for response
self.welcome_msg = self.socket.recv(self.buffer_size) # receive welcome message
from joblib import Parallel, delayed
from multiprocessing import cpu_count
try:
_pools = cpu_count()
except NotImplementedError:
_pools = 4
# compress
import json
from PIL import Image
from PIL.ImagePalette import ImagePalette
from copy import copy
# debug with `DEBUG=leicaexperiment python script.py`
debug = pydebug.debug('leicaexperiment')
# glob for consistent cross platform behavior
def glob(pattern):
"Sorted glob."
from glob import glob as sysglob
return sorted(sysglob(pattern))
# variables in case custom folders
_slide = 'slide'
_chamber = 'chamber'
_field = 'field'
_image = 'image'
_additional_data = 'AdditionalData'