def fork_tensorflow_server():
import atexit
import os
import select
import signal
import subprocess
import sys
import time
import DDFacet
import DDFacet.Other.MyLogger as MyLogger
log=MyLogger.getLogger(" TensorflowServerFork").logger
script = '{d}/tf_server_fork.py'.format(d=os.path.dirname(DDFacet.__file__))
stderr_file = open('tf_err.log', 'w')
proc = subprocess.Popen([sys.executable, script],
preexec_fn=os.setsid,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=stderr_file)
# Register a handler to kill the child process on exit
atexit.register(lambda p: os.kill(p.pid, signal.SIGKILL), proc)
try:
# Wait for 10 seconds to see if the child stdout is ready to read
r, w, e = select.select([proc.stdout], [], [], 10)
# No output was received, fall over
if proc.stdout not in r:
raise ValueError("Tensorflow child process "
"did not return it's server target.")
# Some output was received, try and get the server target
tf_server_target = proc.stdout.readline().rstrip()
# Do some sanity checking here
if not tf_server_target.startswith("grpc://localhost"):
raise ValueError("Tensorflow child process did not return a "
"valid server target. Received '%s'." % tf_server_target)
except ValueError:
# Log exception, kill the child process and rethrow to fall over
log.exception('Exception spawning tensorflow server:')
os.kill(proc.pid, signal.SIGKILL)
raise
else:
log.info("Tensorflow server target is '%s'" % tf_server_target)
return tf_server_target
import psutil
import os, re, errno
import Queue
import multiprocessing
import numpy as np
import glob
import re
from DDFacet.Other import MyLogger
from DDFacet.Other import ClassTimeIt
from DDFacet.Other import ModColor
from DDFacet.Other.progressbar import ProgressBar
from DDFacet.Array import NpShared
from DDFacet.Array import shared_dict
log = MyLogger.getLogger("Multiprocessing")
#MyLogger.setSilent("Multiprocessing")
# store PID here, so that it corresponds to the PID of the parent thread (when we fork off child processes)
_pid = os.getpid()
def getShmPrefix():
"""Returns prefix used for shared memory arrays. ddf.PID is the convention"""
return "ddf.%d" % _pid
# init SharedDict with the same base name
shared_dict.SharedDict.setBaseName(getShmPrefix())
def getShmName(name, **kw):
"""
Forms up a name for a shm-backed shared element. This takes the form of "ddf.PID.", where PID is the
# import DDFacet.cbuild.Gridder._pyGridderSmearPolsFaster as _pyGridderSmearFaster
##########################################################"
# Please do not remove this import again - Needed for killMS
import DDFacet.cbuild.Gridder._pyGridder as _pyGridder
##########################################################"
import numpy as np
import os
import ModCF
from DDFacet.ToolsDir.ModToolBox import EstimateNpix
from DDFacet.ToolsDir import ModFFTW
from DDFacet.Parset import ReadCFG
from DDFacet.Other import ClassTimeIt
from DDFacet.Data import ClassVisServer
from DDFacet.Other import MyLogger
log = MyLogger.getLogger("ClassDDEGridMachine")
def testGrid():
import pylab
# Parset=ReadCFG.Parset("%s/Parset/DefaultParset.cfg"%os.environ["DDFACET_DIR"])
Parset = ReadCFG.Parset("%s/DDFacet/Parset/DefaultParset.cfg" %
os.environ["DDFACET_DIR"])
DC = Parset.DicoPars
# 19 (-0.01442078294460315, 0.014406238534169863) 2025 3465 -10.0
#(array([0]), array([0]), array([1015]), array([1201]))
#(array([0]), array([0]), array([1050]), array([1398]))
# 17 (-0.014391694123736577, 0.01437714971330329) 2025 3465 -10.0
#(array([0]), array([0]), array([1030]), array([1303]))
import scipy
import numpy as np
import pyfftw
from DDFacet.Array import NpShared
from DDFacet.Other import ClassTimeIt
import psutil
import numexpr
from DDFacet.Other.AsyncProcessPool import APP
from DDFacet.Array import shared_dict
from DDFacet.Other import MyLogger
import ModToolBox
from DDFacet.ToolsDir.ModToolBox import EstimateNpix
from DDFacet.ToolsDir import Gaussian
log = MyLogger.getLogger("ModFFTW")
#Fs=pyfftw.interfaces.numpy_fft.fftshift
#iFs=pyfftw.interfaces.numpy_fft.ifftshift
Fs = scipy.fftpack.fftshift
iFs = scipy.fftpack.ifftshift
NCPU_global = 0 #psutil.cpu_count()
#def test():
# size=20
# dtype=np.complex128
# test_array = np.zeros( (size,size), dtype=dtype)
#
# test_array[11,11]=1
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
'''
import re
import numpy as np
from DDFacet.Other import MyLogger
log = MyLogger.getLogger("ClassStokes")
'''
Enumeration of stokes and correlations used in MS2.0 - as per Stokes.h in casacore, the rest are left unimplemented:
These are useful when working with visibility data (https://casa.nrao.edu/Memos/229.html#SECTION000613000000000000000)
'''
StokesTypes = {
'I': 1,
'Q': 2,
'U': 3,
'V': 4,
'RR': 5,
'RL': 6,
'LR': 7,
'LL': 8,
'XX': 9,
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
'''
import numpy as np
from DDFacet.Other import MyLogger
from DDFacet.Other import ModColor
log = MyLogger.getLogger("MaskMachine")
from pyrap.images import image
import scipy.special
import copy
from DDFacet.Imager.ModModelMachine import ClassModModelMachine
from DDFacet.ToolsDir import ModFFTW
class ClassImageNoiseMachine():
def __init__(self,
GD,
ExternalModelMachine=None,
DegridFreqs=None,
GridFreqs=None,
MainCache=None):
import pyfits
import numpy as np
import time
from SkyModel.Other import ModColor
import scipy.ndimage
from SkyModel.Other.progressbar import ProgressBar
import findrms
from DDFacet.Other import MyLogger
log = MyLogger.getLogger("ClassIsland")
class ClassIslands():
def __init__(self,
A,
T=None,
box=(100, 100),
MinPerIsland=4,
DeltaXYMin=2,
Boost=3,
DoPlot=False,
FillNoise=True,
MaskImage=None):
self.A = A
self.T = T
self.MaskImage = MaskImage
self.Noise = None
self.box = box
self.MinPerIsland = MinPerIsland
self.DeltaXYMin = DeltaXYMin
self.FitIm = None
import itertools
import numpy as np
from DDFacet.Other import MyLogger
from DDFacet.Other import ClassTimeIt
from DDFacet.Other import ModColor
log=MyLogger.getLogger("ClassModelMachineHogbom")
from DDFacet.Array import NpParallel
from DDFacet.Array import ModLinAlg
from DDFacet.ToolsDir import ModFFTW
from DDFacet.ToolsDir import ModToolBox
from DDFacet.Other import ClassTimeIt
from DDFacet.Other import MyPickle
from DDFacet.Other import reformat
from DDFacet.ToolsDir.GiveEdges import GiveEdges
from DDFacet.Imager import ClassModelMachine as ClassModelMachinebase
from DDFacet.Imager import ClassFrequencyMachine
import scipy.ndimage
from SkyModel.Sky import ModRegFile
from pyrap.images import image
from SkyModel.Sky import ClassSM
import os
class ClassModelMachine(ClassModelMachinebase.ClassModelMachine):
def __init__(self,*args,**kwargs):
ClassModelMachinebase.ClassModelMachine.__init__(self, *args, **kwargs)
self.DicoSMStacked={}
self.DicoSMStacked["Type"]="Hogbom"
def setRefFreq(self, RefFreq, Force=False):
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
'''
#import sharedarray.SharedArray as SharedArray
import SharedArray
from DDFacet.Other import ModColor
import numpy as np
from DDFacet.Other import MyLogger
import traceback
log = MyLogger.getLogger("NpShared")
import os.path
def zeros(Name, *args, **kwargs):
try:
return SharedArray.create(Name, *args, **kwargs)
except:
DelArray(Name)
return SharedArray.create(Name, *args, **kwargs)
def SizeShm():
from subprocess import check_output
import subprocess
try:
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
'''
import numpy as np
from scipy.interpolate import interp1d
import pandas as pd
from pyrap import quanta as qa
import datetime
from DDFacet.Data.ClassStokes import ClassStokes
from DDFacet.Other import MyLogger
log = MyLogger.getLogger("PointingProvider")
class InvalidPointingSolutions(Exception):
pass
class InvalidFeedType(Exception):
pass
class PointingProvider(object):
__COMPULSORY_HEADER = [
"#ANT", "POL", "TIME", "RA_ERR(deg)", "DEC_ERR(deg)"
]
__AVAILABLE_INTERPOLATORS = {
import numpy as np
from DDFacet.Other import ClassTimeIt
from DDFacet.Other import MyLogger
log = MyLogger.getLogger("ClassConvMatrix")
import scipy.signal
from DDFacet.Array import ModLinAlg
from DDFacet.ToolsDir.GiveEdges import GiveEdgesDissymetric
from DDFacet.ToolsDir import ModFFTW
from DDFacet.ToolsDir.ModToolBox import EstimateNpix
def test():
import DDFacet.ToolsDir.Gaussian
_, _, PSF = DDFacet.ToolsDir.Gaussian.Gaussian(10, 311, 1.)
#PSF.fill(1.)
#import scipy.signal
#PP=scipy.signal.fftconvolve(PSF,PSF, mode='same')
#print Fact
import pylab
pylab.clf()
pylab.imshow(PSF, interpolation="nearest")
pylab.colorbar()
pylab.draw()
pylab.show(False)
pylab.pause(0.1)
Dirty = np.zeros_like(PSF)
nx, _ = Dirty.shape
Dirty[nx / 2, nx / 2 + 10] += 2.
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
"""
from __future__ import print_function
#!/usr/bin/env python
from builtins import str
import optparse
import pickle
import numpy as np
import numpy as np
#import pylab
import os
from DDFacet.Other import MyLogger
from DDFacet.Other import MyPickle
from DDFacet.Other import ModColor
log = MyLogger.getLogger("ClassInterpol")
IdSharedMem = str(int(os.getpid())) + "."
from pyrap.tables import table
from killMS.Other.ClassTimeIt import ClassTimeIt
from killMS.Other.least_squares import least_squares
import copy
import astropy.io.fits as pyfits
import glob
import surveys_db
SaveName = "last_InterPol.obj"
def read_options():
desc = """Questions and suggestions: [email protected]"""
global options
from pyrap.images import image
import numpy as np
from DDFacet.Imager import ClassCasaImage
#from DDFacet.Imager.ModModelMachine import GiveModelMachine
from DDFacet.Imager.ModModelMachine import ClassModModelMachine
from DDFacet.Other import MyLogger
from DDFacet.ToolsDir import ModFFTW
from DDFacet.Other import AsyncProcessPool
from DDFacet.Other.AsyncProcessPool import APP, WorkerProcessError
from DDFacet.Other import Multiprocessing
from DDFacet.Other import MyLogger
from DDFacet.Other import MyPickle
from DDFacet.ToolsDir.rad2hmsdms import rad2hmsdms
log = MyLogger.getLogger("ClassRestoreMachine")
import scipy.signal
import scipy.stats
import multiprocessing
NCPU_default = str(int(0.75 * multiprocessing.cpu_count()))
def read_options():
desc = """DDFacet """
opt = optparse.OptionParser(usage='Usage: %prog <options>',
version='%prog version 1.0',
description=desc)
group = optparse.OptionGroup(opt, "* Data selection options")
group.add_option('--BaseImageName', help='')
import numpy as np
import copy
import time
import traceback
from DDFacet.Imager.MSMF import ClassImageDeconvMachineMSMF
from DDFacet.ToolsDir.GiveEdges import GiveEdges
from DDFacet.Imager.ModModelMachine import ClassModModelMachine
from DDFacet.Other import ClassTimeIt
from DDFacet.Other import MyLogger
log = MyLogger.getLogger("ClassInitSSDModel")
from DDFacet.Other import ModColor
from ClassConvMachine import ClassConvMachineImages
from DDFacet.Imager import ClassMaskMachine
from DDFacet.Array import shared_dict
import psutil
from DDFacet.Other.progressbar import ProgressBar
from DDFacet.Other.AsyncProcessPool import APP
class ClassInitSSDModelParallel():
def __init__(self,
GD,
NFreqBands,
RefFreq,
MainCache=None,
IdSharedMem=""):
self.GD = GD
self.InitMachine = ClassInitSSDModel(GD, NFreqBands, RefFreq,
MainCache, IdSharedMem)
self.NCPU = (self.GD["Parallel"]["NCPU"] or psutil.cpu_count())
import numpy as np
import scipy.signal
import ClassConvMachine
from DDFacet.Other import MyLogger
log=MyLogger.getLogger("ClassSmearSM")
import multiprocessing
from DDFacet.Array import NpShared
from DDFacet.Array import ModLinAlg
import time
from DDFacet.Other.progressbar import ProgressBar
class ClassSmearSM():
def __init__(self,MeanResidual,MeanModelImage,PSFServer,DeltaChi2=4.,IdSharedMem="",NCPU=6):
IdSharedMem+="SmearSM."
NpShared.DelAll(IdSharedMem)
self.IdSharedMem=IdSharedMem
self.NCPU=NCPU
self.MeanModelImage=NpShared.ToShared("%sMeanModelImage"%self.IdSharedMem,MeanModelImage)
self.MeanResidual=NpShared.ToShared("%sMeanResidual"%self.IdSharedMem,MeanResidual)
NPixStats=10000
RandomInd=np.int64(np.random.rand(NPixStats)*(MeanResidual.size))
self.RMS=np.std(np.real(self.MeanResidual.ravel()[RandomInd]))
self.FWHMMin=3.
self.PSFServer=PSFServer
self.DeltaChi2=DeltaChi2
self.Var=self.RMS**2
self.NImGauss=31
self.CubeMeanVariablePSF=NpShared.ToShared("%sCubeMeanVariablePSF"%self.IdSharedMem,self.PSFServer.DicoVariablePSF['CubeMeanVariablePSF'])
self.DicoConvMachine={}
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
'''
import numpy as np
from DDFacet.Other import MyLogger
from DDFacet.Other import ClassTimeIt
from DDFacet.Other import ModColor
log = MyLogger.getLogger("ClassModelMachineSSD")
from DDFacet.Array import NpParallel
from DDFacet.Array import ModLinAlg
from DDFacet.ToolsDir import ModFFTW
from DDFacet.ToolsDir import ModToolBox
from DDFacet.Other import ClassTimeIt
from DDFacet.Other import MyPickle
from DDFacet.Other import reformat
from DDFacet.Imager import ClassFrequencyMachine
from DDFacet.ToolsDir.GiveEdges import GiveEdges
from DDFacet.Imager import ClassModelMachine as ClassModelMachinebase
from DDFacet.ToolsDir import ModFFTW
import scipy.ndimage
from SkyModel.Sky import ModRegFile
from pyrap.images import image
from SkyModel.Sky import ClassSM
from SkyModel.Sky import ModVoronoi
from DDFacet.Other import reformat
import os
from DDFacet.ToolsDir.ModToolBox import EstimateNpix
from DDFacet.Other import ModColor
import tables
from DDFacet.Imager.ClassImToGrid import ClassImToGrid
from matplotlib.path import Path
from SkyModel.Sky import ModVoronoiToReg
import Polygon
from DDFacet.ToolsDir import rad2hmsdms
from DDFacet.Other.ClassTimeIt import ClassTimeIt
from DDFacet.Other import MyLogger
log = MyLogger.getLogger("ClassFacetMachineTessel")
MyLogger.setSilent("MyLogger")
#from DDFacet.cbuild.Gridder import _pyGridderSmearPols as _pyGridderSmear
class ClassFacetMachineTessel(ClassFacetMachine.ClassFacetMachine):
"""
This class extends the basic infrastructure set out in ClassFacetMachine to
split the sky into a Voronoi tesselated sky. The projection, unprojection and resampling logic
remains the same.
"""
def __init__(self, *args, **kwargs):
ClassFacetMachine.ClassFacetMachine.__init__(self, *args, **kwargs)
def setFacetsLocs(self):
NFacets = self.NFacets
#!/usr/bin/env python
import numpy as np
from DDFacet.Other import MyLogger
log = MyLogger.getLogger("MakeCatalog")
#import pyfits
import optparse
import bdsf
import psutil
import pickle
import os
SaveFile = "MakeCatalog.last"
from DDFacet.Other import ModColor
def read_options():
desc = """Make catalog"""
opt = optparse.OptionParser(usage='Usage: %prog <options>',
version='%prog version 1.0',
description=desc)
group = optparse.OptionGroup(opt, "* Data selection options")
group.add_option('--RestoredIm', type=str, help='', default=None)
group.add_option('--bdsm_thresh_isl', type='float', help='', default=10.)
group.add_option('--bdsm_thresh_pix', type='float', help='', default=10.)
group.add_option('--bdsm_rms_box', type=str, help='', default='30,10')
group.add_option('--rmsmean_map', type=str, help='', default='')
group.add_option('--Parallel', type=int, help='', default=1)
group.add_option('--NCPU', type=int, help='', default=0)
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
'''
import os, os.path, subprocess
import cPickle
import collections
from DDFacet.Other import MyLogger, ModColor
log = MyLogger.getLogger("CacheManager")
class CacheManager(object):
"""
# CacheManager
This is a simple disk-based cache management class. Usage is as follows:
cache = CacheManager("/cache/directory")
This creates a cache based on the specified path. Directory is created if it doesn't exist.
If you create CacheManager with reset=True, the contents of the directory are deleted.
Typically, you would use the cache in the following pattern:
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
'''
import ClassModelMachine
import ClassGainMachine
from DDFacet.Other import MyPickle
from DDFacet.Other import MyLogger
from DDFacet.Other import ModColor
log = MyLogger.getLogger("GiveModelMachine")
class ClassModModelMachine():
"""
This is the factory class for ModelMachine. Basically give it a dictionary containing the components of a model image
and it instantiates and returns a copy of the correct ModelMachine. Each pickled dictionary should contain a field
labelling which deconvolution algorithm it corresponds to.
"""
def __init__(self, GD=None):
"""
Input:
GD = Global dictionary
"""
self.GD = GD
self.SSDMM = None
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
'''
import numpy as np
import gaussfitter2
import scipy.ndimage.measurements
from DDFacet.Other import MyLogger
log = MyLogger.getLogger("FitPSF")
def FitCleanBeam(PSF):
"""
Does a least squares fit of an eliptical gaussian to the PSF to determine the
parameters for the clean beam.
Returns:
sigma_x of gaussian in pixels
sigma_y of gaussian in pixels
theta: The rotation angle (radians) of the gaussian from the y axis counter-clockwise (beware of the periodicity)
"""
# zero everything except the main lobe
PSF1 = PSF.copy()
labels = scipy.ndimage.measurements.label(PSF1 >= 0)[0]
import random
import numpy
import numpy as np
from deap import algorithms
import DeapAlgo as algorithms
from deap import base
from deap import creator
from deap import tools
import pylab
from scipy.spatial import Voronoi
import ModVoronoi
from DDFacet.Other import MyLogger
log = MyLogger.getLogger("ClusterDEAP")
from DDFacet.Other import ClassTimeIt
#from scoop import futures
import multiprocessing
import scipy.stats
import Polygon
import ClassMetricDEAP
def test():
Np = 1000
x = np.random.randn(Np)
y = np.random.randn(Np)
CC = ClassCluster(x, y)
CC.Cluster()
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
'''
import optparse
from DDFacet.Array import NpShared
from DDFacet.Other import MyLogger
log = MyLogger.getLogger("ClearSHM")
from DDFacet.cbuild.Gridder import _pyGridderSmearPols as _pyGridderSmear
import glob
import os
import shutil
from DDFacet.Other import Multiprocessing
def read_options():
desc = """CohJones Questions and suggestions: [email protected]"""
opt = optparse.OptionParser(usage='Usage: %prog <options>',
version='%prog version 1.0',
description=desc)
from Other import ModColor
import pickle
import optparse
#from PSourceExtract.ClassPointFit2 import ClassPointFit as ClassFit
#import ClassPointFit as ClassPointFit
from PSourceExtract import ClassFitIslands
from pyrap.images import image
from Other.progressbar import ProgressBar
from Other import reformat
from Sky import ClassSM
from Other import rad2hmsdms
from DDFacet.Other import MyLogger
log = MyLogger.getLogger("Gaussify")
def read_options():
desc = """Questions and suggestions: [email protected]"""
global options
opt = optparse.OptionParser(
usage='Usage: %prog --ms=somename.MS <options>',
version='%prog version 1.0',
description=desc)
group = optparse.OptionGroup(opt, "* Data-related options",
"Won't work if not specified.")
group.add_option('--RestoredImage', help='', type="str", default="")
group.add_option('--MaskName', help='', type="str", default="")
import signal
from collections import OrderedDict
import glob
import re
import numexpr
import time
from DDFacet.Other import MyLogger
from DDFacet.Other import ClassTimeIt
from DDFacet.Other import ModColor
from DDFacet.Other import Exceptions
from DDFacet.Other.progressbar import ProgressBar
from DDFacet.Array import shared_dict
import DDFacet.cbuild.Gridder._pyArrays as _pyArrays
log = MyLogger.getLogger("AsyncProcessPool")
SIGNALS_TO_NAMES_DICT = dict((getattr(signal, n), n) \
for n in dir(signal) if n.startswith('SIG') and '_' not in n )
# PID of parent process
parent_pid = os.getpid()
# Exception type for worker process errors
class WorkerProcessError(Exception):
pass
class Job(object):
def __init__(self,
from pyrap.images import image
from Sky import ClassSM
import optparse
import numpy as np
import glob
import os
from Other import reformat
SaveFile = "last_MyCasapy2BBS.obj"
import pickle
import scipy.ndimage
from Tools import ModFFTW
from SkyModel.PSourceExtract import ClassIslands
from SkyModel.Other.ClassCasaImage import PutDataInNewImage
import scipy.special
from DDFacet.Other import MyLogger
log = MyLogger.getLogger("MakeMask")
from SkyModel.Other.progressbar import ProgressBar
import collections
from SkyModel.Other.MyHist import MyCumulHist
from SkyModel.PSourceExtract import Gaussian
from SkyModel.Sky import ModRegFile
from DDFacet.ToolsDir import ModFFTW
import DDFacet.Imager.SSD.ClassIslandDistanceMachine
from DDFacet.ToolsDir.rad2hmsdms import rad2hmsdms
from DDFacet.Other import MyPickle
def read_options():
desc = """ [email protected]"""
opt = optparse.OptionParser(
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
'''
import itertools
import numpy as np
from DDFacet.Other import MyLogger
from DDFacet.Other import ClassTimeIt
from DDFacet.Other import ModColor
log = MyLogger.getLogger("ClassModelMachineMSMF")
from DDFacet.Array import NpParallel
from DDFacet.Array import ModLinAlg
from DDFacet.ToolsDir import ModFFTW
from DDFacet.ToolsDir import ModToolBox
from DDFacet.Other import ClassTimeIt
from DDFacet.Other import MyPickle
from DDFacet.Other import reformat
from DDFacet.Imager import ClassFrequencyMachine
from DDFacet.ToolsDir.GiveEdges import GiveEdges
from DDFacet.ToolsDir.GiveEdges import GiveEdgesDissymetric
from DDFacet.Imager import ClassModelMachine as ClassModelMachinebase
from DDFacet.ToolsDir import ModFFTW
import scipy.ndimage
from SkyModel.Sky import ModRegFile
from pyrap.images import image
import collections
import random
import numpy as np
from DDFacet.Other import ClassTimeIt
from DDFacet.Other import MyLogger
log = MyLogger.getLogger("ClassArrayMethodSSD")
import multiprocessing
import ClassConvMachine
import time
from scipy.stats import chi2
from deap import tools
log = MyLogger.getLogger("ClassArrayMethodSSD")
from ClassParamMachine import ClassParamMachine
from DDFacet.ToolsDir.GeneDist import ClassDistMachine
import ClassMutate
class ClassArrayMethodSSD():
def __init__(self,
Dirty,
PSF,
ListPixParms,
ListPixData,
FreqsInfo,
GD=None,
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
'''
"""
This minimal implementation of standard Hogbom CLEAN algorithm should serve
as a minimal reference interface of how to incorporate new deconvolution
algorithms into DDFacet.
"""
import numpy as np
import numexpr
from DDFacet.Other import MyLogger
from DDFacet.Other import ModColor
log = MyLogger.getLogger("ClassImageDeconvMachine")
from DDFacet.Array import NpParallel
from DDFacet.Other import ClassTimeIt
from pyrap.images import image
from DDFacet.Imager.ClassPSFServer import ClassPSFServer
from DDFacet.Other.progressbar import ProgressBar
from DDFacet.Imager import ClassGainMachine # Currently required by model machine but fixed to static mode
from DDFacet.ToolsDir import GiveEdges
class ClassImageDeconvMachine():
"""
Currently constructor inputs match those in MSMF, should figure out which are truly generic and put the rest in
parset's MinorCycleConfig option.
These methods may be called from ClassDeconvMachine
Init(**kwargs) - contains minor cycle specific initialisations which are only used once
def main(OP=None, messages=[]):
if OP is None:
OP = MyPickle.Load(SaveFile)
print "Using settings from %s, then command line." % SaveFile
DicoConfig = OP.DicoConfig
ImageName = DicoConfig["Output"]["Name"]
if not ImageName:
raise Exceptions.UserInputError(
"--Output-Name not specified, can't continue.")
if not DicoConfig["Data"]["MS"]:
raise Exceptions.UserInputError(
"--Data-MS not specified, can't continue.")
# create directory if it exists
dirname = os.path.dirname(ImageName)
if not os.path.exists(dirname) and not dirname == "":
os.mkdir(dirname)
# setup logging
MyLogger.logToFile(ImageName + ".log", append=DicoConfig["Log"]["Append"])
global log
log = MyLogger.getLogger("DDFacet")
# disable colors and progressbars if requested
ModColor.silent = SkyModel.Other.ModColor.silent = \
progressbar.ProgressBar.silent = \
DicoConfig["Log"]["Boring"]
if messages:
if not DicoConfig["Log"]["Boring"]:
#os.system('clear')
logo.print_logo()
for msg in messages:
print >> log, msg
if DicoConfig["Debug"]["Pdb"] == "always":
print >> log, "--Debug-Pdb=always: unexpected errors will be dropped into pdb"
Exceptions.enable_pdb_on_error(
ModColor.Str(
"DDFacet has encountered an unexpected error. Dropping you into pdb for a post-mortem.\n"
+
"(This is because you're running with --Debug-Pdb set to 'always'.)"
))
elif DicoConfig["Debug"][
"Pdb"] == "auto" and not DicoConfig["Log"]["Boring"]:
print >> log, "--Debug-Pdb=auto and not --Log-Boring: unexpected errors will be dropped into pdb"
Exceptions.enable_pdb_on_error(
ModColor.Str(
"DDFacet has encountered an unexpected error. Dropping you into pdb for a post-mortem.\n"
+
"(This is because you're running with --Debug-Pdb set to 'auto' and --Log-Boring is off.)"
))
# print current options
OP.Print(dest=log)
# enable memory logging
MyLogger.enableMemoryLogging(DicoConfig["Log"]["Memory"])
# get rid of old shm arrays from previous runs
Multiprocessing.cleanupStaleShm()
# initialize random seed from config if set, or else from system time
if DicoConfig["Misc"]["RandomSeed"] is not None:
print >> log, "random seed=%d (explicit)" % DicoConfig["Misc"][
"RandomSeed"]
else:
DicoConfig["Misc"]["RandomSeed"] = int(time.time())
print >> log, "random seed=%d (automatic)" % DicoConfig["Misc"][
"RandomSeed"]
np.random.seed(DicoConfig["Misc"]["RandomSeed"])
# init NCPU for different bits of parallelism
ncpu = DicoConfig["Parallel"]["NCPU"] or psutil.cpu_count()
DicoConfig["Parallel"]["NCPU"] = ncpu
_pyArrays.pySetOMPNumThreads(ncpu)
NpParallel.NCPU_global = ModFFTW.NCPU_global = ncpu
numexpr.set_num_threads(ncpu)
print >> log, "using up to %d CPUs for parallelism" % ncpu
# write parset
OP.ToParset("%s.parset" % ImageName)
Mode = DicoConfig["Output"]["Mode"]
# init semaphores, as they're needed for weight calculation too
ClassFacetMachine.ClassFacetMachine.setup_semaphores(DicoConfig)
# data machine initialized for all cases except PSF-only mode
# psf machine initialized for all cases except Predict-only mode
Imager = ClassDeconvMachine.ClassImagerDeconv(
GD=DicoConfig,
BaseName=ImageName,
predict_only=(Mode == "Predict" or Mode == "Subtract"),
data=(Mode != "PSF"),
psf=(Mode != "Predict" and Mode != "Dirty" and Mode != "Subtract"),
readcol=(Mode != "Predict" and Mode != "PSF"),
deconvolve=("Clean" in Mode))
Imager.Init()
# Imager.testDegrid()
# stop
if "Predict" in Mode or "Subtract" in Mode:
Imager.GivePredict()
if "Clean" in Mode:
Imager.main()
elif "Dirty" in Mode:
sparsify = DicoConfig["Comp"]["Sparsification"]
if sparsify and isinstance(sparsify, list):
sparsify = sparsify[0]
Imager.GiveDirty(psf="PSF" in Mode, sparsify=sparsify)
elif "PSF" in Mode:
sparsify = DicoConfig["Comp"]["Sparsification"]
if sparsify and isinstance(sparsify, list):
sparsify = sparsify[0]
Imager.MakePSF(sparsify=sparsify)
elif "RestoreAndShift" == Mode:
Imager.RestoreAndShift()