class Main(QWidget):
def __init__(self):
super().__init__()
self.initUI()
def initUI(self):
desing_of_main_menu(self)
def StbI(self):
self.stbi = STByID(self)
self.stbi.show()
self.close()
def Prog(self):
self.prog = Progress(self)
self.prog.show()
self.close()
def RanNum(self):
self.rannum = Random_Number(self)
self.rannum.show()
self.close()
def Option(self):
self.option = Get_Opton(self)
self.option.show()
self.close()
def response(self, response, url):
"""Response handler"""
self.counter.update(("completed", ))
if hasattr(response, 'status'):
if response.status in self.DEFAULT_HTTP_FAILED_STATUSES:
self.iterator = Progress.line(url, self.urls.__len__(),
'error', self.iterator)
self.counter.update(("failed", ))
elif response.status in self.DEFAULT_HTTP_SUCCESS_STATUSES:
self.iterator = Progress.line(url, self.urls.__len__(),
'success', self.iterator)
self.counter.update(("success", ))
elif response.status in self.DEFAULT_HTTP_UNRESOLVED_STATUSES:
# self.iterator = Progress.line(url, self.urls.__len__(), 'warning', self.iterator)
self.counter.update(("possible", ))
elif response.status in self.DEFAULT_HTTP_REDIRECT_STATUSES:
# self.iterator = Progress.line(url, self.urls.__len__(), 'warning', self.iterator)
self.counter.update(("redirects", ))
else:
self.counter.update(("undefined", ))
return
self.result[response.status].append(url)
else:
return
def __init__(self, root):
self.root = root
self.img_dir = ""
self.perSec = -1
self.params_json = ""
self.diff_dirs = ["", ""]
self.track_json = ""
self.contour_json = ""
self.window = tk.Toplevel(self.root)
self.window.geometry("350x200")
self.l0 = tk.Label(
self.window,
text="Select the video to process into frames,\n"
"entering the number of frames per second to extract\n"
"(default 0 means extract all frames available)\n"
"-or- select the directory containing frames:")
self.e0 = tk.Entry(self.window, width=4)
self.e0.insert(0, "0")
self.b0 = tk.Button(self.window,
text="Select video",
command=self.getFrames)
self.l1 = tk.Label(self.window, text="-or-")
self.b1 = tk.Button(self.window,
text="Select folder",
command=self.getFolder)
self.prog = Progress(self.window)
self.l0.pack()
self.e0.pack()
self.b0.pack()
self.l1.pack()
self.b1.pack()
def delete(self):
"""Delete the VM.
VM must be locked or unregistered"""
with VirtualBoxException.ExceptionHandler():
iMachine = self.getIMachine()
iprogress = iMachine.delete(None)
progress = Progress(iprogress)
progress.waitForCompletion()
def url_action(self, url, params=()):
""" Load by url action """
result = Http().get(url, params)
if result:
Progress.view(result)
is_logging = params.get('log', self.DEFAULT_LOGGING)
if is_logging:
log.syslog(url, result)
exit()
def createBaseStorage(self, size, variant=None, wait=True):
"""Create storage for the drive of the given size (in MB).
Returns Progress instance. If wait is True, does not return until process completes."""
if variant is None:
variant = Constants.MediumVariant_Standard
with VirtualBoxException.ExceptionHandler():
progress = self.getIMedium().createBaseStorage(size, variant)
progress = Progress(progress)
if wait:
progress.waitForCompletion()
return progress
def takeSnapshot(self, name, description=None, wait=True):
"""Saves the current execution state and all settings of the machine and creates differencing images for all normal (non-independent) media.
Returns Progress instance. If wait is True, does not return until process completes."""
assert(name is not None)
with self.lock() as session:
with VirtualBoxException.ExceptionHandler():
iprogress = session.console.takeSnapshot(name, description)
progress = Progress(iprogress)
# XXX Not sure if we need a lock for this or not
if wait:
progress.waitForCompletion()
return progress
def deleteSnapshot(self, snapshot, wait=True):
"""Deletes the specified snapshot.
Returns Progress instance. If wait is True, does not return until process completes."""
assert(snapshot is not None)
with self.lock() as session:
with VirtualBoxException.ExceptionHandler():
iprogress = session.console.deleteSnapshot(snapshot.id)
progress = Progress(iprogress)
# XXX Not sure if we need a lock for this or not
if wait:
progress.waitForCompletion()
return progress
def cloneTo(self, target, variant=None, parent=None, wait=True):
"""Clone to the target hard drive.
Returns Progress instance. If wait is True, does not return until process completes."""
if variant is None:
variant = Constants.MediumVariant_Standard
with VirtualBoxException.ExceptionHandler():
progress = self.getIMedium().cloneTo(target.getIMedium(),
variant,
parent)
progress = Progress(progress)
if wait:
progress.waitForCompletion()
return progress
def powerOn(self, type="gui", env=""):
"""Spawns a new process that executes a virtual machine.
This is spawning a "remote session" in VirtualBox terms."""
# TODO: Add a wait argument
if not self.isRegistered():
raise VirtualBoxException.VirtualBoxInvalidVMStateException(
"VM is not registered")
with VirtualBoxException.ExceptionHandler():
iMachine = self.getIMachine()
session = Session.create()
iprogress = iMachine.launchVMProcess(session.getISession(),
type, env)
progress = Progress(iprogress)
progress.waitForCompletion()
session.unlockMachine()
def request(self, url):
"""Request handler"""
# if True == self.proxy:
# proxyserver = self.reader.get_random_proxy()
# try:
# conn = urllib3.proxy_from_url(proxyserver, )
# except urllib3.exceptions.ProxySchemeUnknown as e:
# log.critical(e.message + ": " + proxyserver)
# else:
# conn = urllib3.connection_from_url(url, )
try:
response = self.http.urlopen(self.DEFAULT_HTTP_METHOD,
url,
headers=self.HEADER,
redirect=False,
timeout=self.rest,
release_conn=True)
except (urllib3.exceptions.ConnectTimeoutError,
urllib3.exceptions.MaxRetryError,
urllib3.exceptions.HostChangedError,
urllib3.exceptions.ReadTimeoutError,
urllib3.exceptions.ProxyError) as e:
response = None
self.iterator = Progress.line(url + ' -> ' + e.message,
self.urls.__len__(), 'warning',
self.iterator)
except exceptions.AttributeError as e:
log.critical(e.message)
except TypeError as e:
log.critical(e.message)
time.sleep(self.delay)
return self.response(response, url)
def __init__(self, inputdata, outputdata):
dao = DAOPsql('furman')
self.geo = GeoSearch(dao)
self.error_log = ErrorLog(self.__class__.__name__)
self.progress = Progress()
self.input = inputdata
self.output = outputdata
def from_json(path, progress, args={}):
"""Returns an Experiment object from a JSON configuration"""
logger.info(path)
shutil.copy(path, op.join(paths.OUTPUT_DIR, 'config.json'))
config = util.load_json(path)
if "test" in args and args["test"] > 0:
config['duration'] = args["test"]
config['time_between_run'] = 1000
config['replications'] = 1
print(
"Test mode enabled (dureation: %d, time_between_run: %d, replications: %d)"
% (config['duration'], config['time_between_run'],
config['replications']))
experiment_type = config['type']
if experiment_type == 'plugintest':
return PluginTests(config)
if progress is None:
progress = Progress(config_file=path,
config=config,
load_progress=False)
restart = False
else:
restart = True
if experiment_type == 'native':
return NativeExperiment(config, progress, restart)
elif experiment_type == 'web':
return WebExperiment(config, progress, restart)
else:
return Experiment(config, progress, restart)
def addfolder(self):
try:
path = self._input('Absolute path : ')
c = Progress("Files add")
self.vpn.index.add_folder(path, c)
except Exception as e:
print "\n[Error]",e
else:
print "\n[+] Folder added : %s" % os.path.abspath(path)
def __replace_color_slow(self, ply_data):
"""
Перемещает цвет пикселя из растра в сопоставленную по координатам точку из облака точек без ускорения numba \n
Для более быстрого переноса использовать "replace_color_from_one_channel_parallel"
"""
i = 0
count_matched = 0
data_count = ply_data[Element.VERTEX.value].count
progress = Progress(data_count)
band_count = self.raster.RasterCount
if band_count == 1:
write_color = self.__write_color_from_one_channel_raster
else:
write_color = self.__write_color_from_three_channel_raster
while i < data_count:
cloud_point_x = ply_data[Element.VERTEX.value].data[Property.X.value][i]
cloud_point_y = ply_data[Element.VERTEX.value].data[Property.Y.value][i]
x, y = self.__find_appropriate_pixel(cloud_point_x, cloud_point_y)
if x != PixelPosition.NOT_MATCHED.value and y != PixelPosition.NOT_MATCHED.value:
count_matched += 1
write_color(ply_data, i, x, y)
progress.step()
i += 1
progress.reset()
print('Всего точек:', data_count)
count_mismatched = data_count - count_matched
return count_matched, count_mismatched
def CreateLiveBlob(file_name):
with ProfileDb(file_name) as profileDb:
profileDb.Clear()
stringMap = StringMap()
progress = Progress()
for section in Progress.SECTIONS:
profileDb.FillSection(section,progress.getIds(section))
profileDb.FillStrings("Fetishes",stringMap.getSection("Fetish"))
pids = set(progress.getIds("CompletedProfiles"))
sys.stderr.write("Profiles to load: [%s]\n" % len(pids))
ploaded = 0
pfailed = 0
ptotal = len(pids)
for pid in pids:
profile = Profile(pid)
if(profile.load()):
profileDb.AddProfile(profile)
ploaded += 1
sys.stderr.write("Progress - Loaded Profile [%12s], [%12s] of [%12s], [%3s%% Done]\n" % (pid,ploaded,ptotal,100*(ploaded+pfailed)/ptotal))
else:
progress.errorProfile(pid)
pfailed += 1
sys.stderr.write("Progress - Failed Profile [%12s], [%12s] of [%12s], [%s%% Done]\n" % (pid,pfailed,ptotal,100*(ploaded+pfailed)/ptotal))
del profile
gids = set(progress.getIds("CompletedGroups"))
sys.stderr.write("Groups to load: [%s]\n" % len(gids))
gloaded = 0
gfailed = 0
gtotal = len(gids)
for gid in gids:
group = Group(gid)
if(group.load()):
profileDb.AddGroup(group)
gloaded += 1
sys.stderr.write("Progress - Loaded Group [%12s], [%12s] of [%12s], [%3s%% Done]\n" % (gid,gloaded,gtotal,100*(gloaded+gfailed)/gtotal))
else:
progress.errorGroup(gid)
failed += 1
sys.stderr.write("Progress - Failed Group [%12s], [%12s] of [%12s], [%s%% Done]\n" % (gid,gfailed,gtotal,100*(gloaded+gfailed)/gtotal))
del group
sys.stderr.write("Loaded [%d] Profiles [%d] Groups [%d] Errors.\n" % (ploaded,gloaded,pfailed+gfailed))
return profileDb
def from_json(path, progress):
"""Returns an Experiment object from a JSON configuration"""
logger.info(path)
shutil.copy(path, op.join(paths.OUTPUT_DIR, 'config.json'))
config = util.load_json(path)
experiment_type = config['type']
if progress is None and not experiment_type == 'test':
progress = Progress(config_file=path, config=config, load_progress=False)
if experiment_type == 'native':
return NativeExperiment(config, progress)
elif experiment_type == 'web':
return WebExperiment(config, progress)
elif experiment_type == 'test':
return Tests(config)
else:
return Experiment(config, progress)
def __init__(self, delegate, parent=None):
QWidget.__init__(self, parent)
self.setupUi(self)
self.delegate = delegate
self.small_screwdriver = DrawBinsWidget()
self.work_layout.insertWidget(0, self.small_screwdriver)
self.progress_window = Progress()
self.settings_window = Settings()
self.bin_packing_thread = BinPackingThread()
self.addDirectory.clicked.connect(self.onAddDirectory)
self.removeImage.clicked.connect(self.onRemoveImages)
self.startPushButton.clicked.connect(self.onStart)
self.methodTabWidget.currentChanged.connect(self.bin_packing_thread.setMethod)
self.binSizeComboBox.currentIndexChanged.connect(self.bin_packing_thread.setBinSize)
self.clearPushButton.clicked.connect(self.delegate.on_remove_all_images)
self.settingsPushButton.clicked.connect(self.settings_window.show)
# self.images_changed.connect(self.updateImages)
self.bin_packing_thread.bin_packing_available.connect(self.startPushButton.setEnabled)
self.bin_packing_thread.on_end.connect(self.startPushButton.setEnabled)
self.bin_packing_thread.on_end.connect(self.progress_window.setHidden)
self.bin_packing_thread.prepare_progress_signal.connect(self.progress_window.prepareProgressBar.setValue)
self.bin_packing_thread.packing_progress_signal.connect(self.progress_window.binPackingProgressBar.setValue)
self.bin_packing_thread.saving_progress_signal.connect(self.progress_window.savingProgressBar.setValue)
self.startPushButton.setEnabled(self.bin_packing_thread.binPackingAvailable())
self.directory = None
self.images = []
self.restoreSettings()
if self.directory:
self.harvestDirectory(self.directory)
self.bin_packing_thread.update_bins.connect(self.small_screwdriver.redrawBins)
self.outputFormatComboBox.currentIndexChanged.connect(BinPackingThread.set_exporter)
BinPackingThread.set_exporter(self.outputFormatComboBox.currentIndex())
def request(self, url):
"""Request handler"""
if True == self.proxy:
proxyserver = self.reader.get_random_proxy()
try:
conn = urllib3.proxy_from_url(proxyserver,
maxsize=10,
block=True,
timeout=self.rest)
except urllib3.exceptions.ProxySchemeUnknown as e:
log.critical(e.message + ": " + proxyserver)
else:
conn = urllib3.connection_from_url(url,
maxsize=10,
block=True,
timeout=self.rest)
headers = {
'accept-encoding': 'gzip, deflate, sdch',
'accept-language':
'ru-RU,ru;q=0.8,en-US;q=0.6,en;q=0.4,uk;q=0.2,es;q=0.2',
'cache-control': 'no-cache',
'user-agent': self.reader.get_random_user_agent()
}
try:
response = conn.request(self.DEFAULT_HTTP_METHOD,
url,
headers=headers)
except (urllib3.exceptions.ConnectTimeoutError,
urllib3.exceptions.MaxRetryError,
urllib3.exceptions.HostChangedError,
urllib3.exceptions.ReadTimeoutError,
urllib3.exceptions.ProxyError) as e:
response = None
self.iterator = Progress.line(url + ' -> ' + e.message,
self.urls.__len__(), 'warning',
self.iterator)
except exceptions.AttributeError as e:
log.critical(e.message)
except TypeError as e:
log.critical(e.message)
time.sleep(self.delay)
return self.response(response, url)
def __check_match_slow(self, ply_data):
"""
Проверка совпадения координат из облака точек с координатами растра без ускорения numba \n
Для более быстрого просчёта использовать "check_match_fast"
"""
i = 0
count_matched = 0
data_count = ply_data[Element.VERTEX.value].count
progress = Progress(data_count)
while i < data_count:
cloud_point_x = ply_data[Element.VERTEX.value].data[Property.X.value][i]
cloud_point_y = ply_data[Element.VERTEX.value].data[Property.Y.value][i]
x, y = self.__find_appropriate_pixel(cloud_point_x, cloud_point_y)
if x != PixelPosition.NOT_MATCHED.value and y != PixelPosition.NOT_MATCHED.value:
count_matched += 1
progress.step()
i += 1
progress.reset()
# print_time('Всего точек:', data_count)
count_mismatched = data_count - count_matched
return count_matched, count_mismatched
raise ExtensionRunError( \
"Error creating repository %s. Exception: %s" % \
(repo.get_uri(), str(e)))
self.__prepare_table(connection)
fp = FilePaths(db)
read_cursor.execute(
statement(
"""select COUNT(*)
from patches p, scmlog s
where p.commit_id = s.id and
s.repository_id = ? and
p.patch is not NULL""", db.place_holder), (repo_id, ))
nr_records = read_cursor.fetchone()[0]
progress = Progress("[Extension Hunks]", nr_records)
patches = self.get_patches(repo, path or repo.get_uri(), repo_id, db,
read_cursor)
for commit_id, file_id, patch_content, rev in patches:
for hunk in self.get_commit_data(patch_content):
insert = """insert into hunks(file_id, commit_id,
old_start_line, old_end_line, new_start_line,
new_end_line)
values(?,?,?,?,?,?)"""
execute_statement(statement(insert, db.place_holder),
(file_id, commit_id, hunk.old_start_line,
hunk.old_end_line, hunk.new_start_line,
hunk.new_end_line),
def __init__(self, root, folder):
self.root = root
self.folder = folder
self.window = tk.Toplevel(self.root)
self.done = False
self.thresh = 10
self.img = None
self.picSize = (0, 0)
self.hist = None
self.histTk = None
self.display = None
self.displayTk = None
self.imgs = sorted(os.listdir(self.folder))
for entry in range(len(self.imgs) - 1, -1, -1):
temp = self.imgs[entry].rsplit(".")
if len(temp) < 2 or temp[-1] not in extension:
self.imgs.pop(entry)
# Make folders necessary for tool
self.diff1_p = self.folder + diff1
self.diff2_p = self.folder + diff2
self.contour = ""
self.params = self.folder + params
if os.path.exists(self.params):
params_f = open(self.params)
self.params_json = json.load(params_f)
params_f.close()
else:
self.params_json = params_json
makedir(self.diff1_p)
makedir(self.diff2_p)
# Advanced parameters
self.canny_lower = self.params_json["canny_lower"]
self.canny_upper = self.params_json["canny_upper"]
self.kernel = self.params_json["kernel"]
self.dilate_it1 = self.params_json["dilate_it1"]
self.erode_it1 = self.params_json["erode_it1"]
self.dilate_it2 = self.params_json["dilate_it2"]
self.erode_it2 = self.params_json["erode_it2"]
self.area_lower = self.params_json["area_lower"]
self.area_upper = self.params_json["area_upper"]
self.setup()
# Make widgets to be placed in
self.l0 = tk.Label(self.window, image=self.displayTk)
self.l1 = tk.Label(self.window, image=self.histTk)
self.slider = tk.Scale(self.window,
from_=0,
to=50,
orient=tk.HORIZONTAL,
length=297)
self.slider.set(self.thresh)
self.l2 = tk.Label(self.window, text="Histogram of Pixel Intensity")
self.b0 = tk.Button(self.window, text="Show", command=self.getPic)
self.b1 = tk.Button(self.window,
text="Set & Run",
command=self.setAndRun)
self.b2 = tk.Button(self.window,
text="Advanced",
command=self.advanced)
self.prog = Progress(self.window)
self.getPic()
self.l0.grid(row=1, column=1, rowspan=5)
self.l1.grid(row=2, column=2, columnspan=3)
self.l2.grid(row=1, column=2, columnspan=3)
self.slider.grid(row=3, column=2, columnspan=3)
self.b0.grid(row=4, column=2)
self.b1.grid(row=4, column=3)
self.b2.grid(row=4, column=4)
class GetThresh:
def __init__(self, root, folder):
self.root = root
self.folder = folder
self.window = tk.Toplevel(self.root)
self.done = False
self.thresh = 10
self.img = None
self.picSize = (0, 0)
self.hist = None
self.histTk = None
self.display = None
self.displayTk = None
self.imgs = sorted(os.listdir(self.folder))
for entry in range(len(self.imgs) - 1, -1, -1):
temp = self.imgs[entry].rsplit(".")
if len(temp) < 2 or temp[-1] not in extension:
self.imgs.pop(entry)
# Make folders necessary for tool
self.diff1_p = self.folder + diff1
self.diff2_p = self.folder + diff2
self.contour = ""
self.params = self.folder + params
if os.path.exists(self.params):
params_f = open(self.params)
self.params_json = json.load(params_f)
params_f.close()
else:
self.params_json = params_json
makedir(self.diff1_p)
makedir(self.diff2_p)
# Advanced parameters
self.canny_lower = self.params_json["canny_lower"]
self.canny_upper = self.params_json["canny_upper"]
self.kernel = self.params_json["kernel"]
self.dilate_it1 = self.params_json["dilate_it1"]
self.erode_it1 = self.params_json["erode_it1"]
self.dilate_it2 = self.params_json["dilate_it2"]
self.erode_it2 = self.params_json["erode_it2"]
self.area_lower = self.params_json["area_lower"]
self.area_upper = self.params_json["area_upper"]
self.setup()
# Make widgets to be placed in
self.l0 = tk.Label(self.window, image=self.displayTk)
self.l1 = tk.Label(self.window, image=self.histTk)
self.slider = tk.Scale(self.window,
from_=0,
to=50,
orient=tk.HORIZONTAL,
length=297)
self.slider.set(self.thresh)
self.l2 = tk.Label(self.window, text="Histogram of Pixel Intensity")
self.b0 = tk.Button(self.window, text="Show", command=self.getPic)
self.b1 = tk.Button(self.window,
text="Set & Run",
command=self.setAndRun)
self.b2 = tk.Button(self.window,
text="Advanced",
command=self.advanced)
self.prog = Progress(self.window)
self.getPic()
self.l0.grid(row=1, column=1, rowspan=5)
self.l1.grid(row=2, column=2, columnspan=3)
self.l2.grid(row=1, column=2, columnspan=3)
self.slider.grid(row=3, column=2, columnspan=3)
self.b0.grid(row=4, column=2)
self.b1.grid(row=4, column=3)
self.b2.grid(row=4, column=4)
def setup(self):
"""Gets the desired threshold for cut-off from user."""
current2 = cv2.imread(self.folder + "/" + self.imgs[0])
for indx in range(1, len(
self.imgs)): # Find a good difference example for user
current1 = current2
current2 = cv2.imread(self.folder + "/" + self.imgs[indx])
diff = cv2.absdiff(current1, current2)
if self.hasContent(diff):
self.img = diff
break
vals = list(range(0, 50))
freq = [
i[0] for i in cv2.calcHist([self.img], [0], None, [256], [0, 256])
][:50]
plt.figure(figsize=(3, 1)).add_axes((0, 0, 1, 1))
plt.bar(vals, freq, color='#0504aa')
plt.axis("off")
buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
self.hist = ImageOps.expand(Image.open(buf), 2, "black")
self.histTk = ImageTk.PhotoImage(self.hist)
self.size()
def hasContent(self, image):
"""Returns true if an image has content above given threshold. False if otherwise."""
_, baw = cv2.threshold(cv2.cvtColor(image, cv2.COLOR_BGR2GRAY),
self.thresh, 255, cv2.THRESH_BINARY)
return bool(np.amax(baw))
def advanced(self):
temp = Advanced(self.window, self.canny_lower, self.canny_upper,
self.kernel, self.dilate_it1, self.erode_it1,
self.dilate_it2, self.erode_it2, self.area_lower,
self.area_upper)
self.window.wait_window(temp.window)
self.canny_lower, self.canny_upper, self.kernel, self.dilate_it1, self.erode_it1, \
self.dilate_it2, self.erode_it2, self.area_lower, self.area_upper = temp.getInfo()
def size(self, scalar=500):
w = int(self.img.shape[1])
h = int(self.img.shape[0])
scale_temp = scalar / w if w > h else scalar / h
self.img = cv2.resize(self.img,
(int(w * scale_temp), int(h * scale_temp)))
self.picSize = (self.img.shape[1], self.img.shape[0])
def getPic(self):
"""Make a new threshold picture based off of the current threshold and put it in the window"""
self.thresh = int(self.slider.get())
(_, baw) = cv2.threshold(cv2.cvtColor(self.img, cv2.COLOR_BGR2GRAY),
self.thresh, 255, cv2.THRESH_BINARY)
self.display = cvToPil(baw)
self.displayTk = ImageTk.PhotoImage(self.display)
self.l0 = tk.Label(self.window, image=self.displayTk)
self.l0.grid(row=1, column=1, rowspan=5)
def selectContours(self, contours):
temp = []
for entry in contours:
if (self.area_upper <= 0 or cv2.contourArea(entry) <= self.area_upper) and \
self.area_lower <= cv2.contourArea(entry):
temp.append(entry)
return temp
def setAndRun(self):
"""Sets threshold based off of slider value and runs the difference"""
if self.thresh != int(self.slider.get()):
self.getPic()
else:
# window configuration
self.b0.config(state=tk.DISABLED)
self.b1.config(state=tk.DISABLED)
self.b2.config(state=tk.DISABLED)
self.prog.grid(row=5, column=2, columnspan=3)
self.prog.config(length=len(self.imgs) * 2 + 10)
# diff 1 files
indx = 1
name2 = self.imgs[0]
current2 = cv2.imread(self.folder + "/" + name2)
length = len(self.imgs)
while indx < length:
name1 = name2
name2 = self.imgs[indx]
current1 = current2
current2 = cv2.imread(self.folder + "/" + name2)
while not self.hasContent(cv2.absdiff(
current1,
current2)): # skip frames w/o significant difference
indx += 1
self.prog.increase()
name2 = self.imgs[indx]
current2 = cv2.imread(self.folder + "/" + name2)
temp = cv2.absdiff(current1,
current2) # difference between pictures
temp = cv2.cvtColor(temp,
cv2.COLOR_BGR2GRAY) # convert to grayscale
temp = cv2.threshold(
temp, self.thresh, 255,
cv2.THRESH_TOZERO)[1] # cut off all pixels below thresh
temp = cv2.GaussianBlur(
temp, (5, 5), 1) # blur to get rid of remaining noise
temp = cv2.Canny(
temp, self.canny_lower,
self.canny_upper) # find edges with Canny algorithm
cv2.imwrite(
self.diff1_p + "/" + name1, temp
) # name is the name of the earlier image in the difference
indx += 1
self.prog.increase()
# diff 2 files
diff1_imgs = sorted(os.listdir(self.diff1_p))
indx = 1
self.contour = self.folder + contour
contour_dict = {}
kernel = np.ones((self.kernel, self.kernel))
current2 = cv2.imread(self.diff1_p + "/" + diff1_imgs.pop(0))
current2 = cv2.dilate(current2, kernel, iterations=self.dilate_it1)
current2 = cv2.erode(current2, kernel, iterations=self.erode_it1)
for image in diff1_imgs:
current1 = current2
current2 = cv2.imread(self.diff1_p + "/" + image)
current2 = cv2.dilate(current2,
kernel,
iterations=self.dilate_it1)
current2 = cv2.erode(current2,
kernel,
iterations=self.erode_it1)
temp = cv2.bitwise_and(current1, current2)
temp = cv2.dilate(temp, kernel, iterations=self.dilate_it2)
temp = cv2.erode(temp, kernel, iterations=self.erode_it2)
temp = cv2.threshold(cv2.cvtColor(temp, cv2.COLOR_BGR2GRAY),
100, 255, cv2.THRESH_BINARY)[1]
# collect and process contours
contours = cv2.findContours(temp, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)[0]
contours = self.selectContours(contours)
contour_dict[image] = contours
# save image with contours
temp = cv2.imread(self.folder + "/" + image)
cv2.drawContours(temp, contours, -1, (0, 255, 0), 3)
cv2.imwrite(self.diff2_p + "/" + image, temp)
indx += 1
self.prog.increase()
f = open(self.contour, "w")
json.dump(contour_dict,
f,
default=lambda obj: obj.tolist()
if isinstance(obj, np.ndarray) else obj)
f.close()
self.setParams()
self.done = True
self.prog.increase(num=10)
self.window.destroy()
def setParams(self):
self.params_json["canny_lower"] = self.canny_lower
self.params_json["canny_upper"] = self.canny_upper
self.params_json["kernel"] = self.kernel
self.params_json["dilate_it1"] = self.dilate_it1
self.params_json["erode_it1"] = self.erode_it1
self.params_json["dilate_it2"] = self.dilate_it2
self.params_json["erode_it2"] = self.erode_it2
self.params_json["area_lower"] = self.area_lower
self.params_json["area_upper"] = self.area_upper
f = open(self.params, "w")
json.dump(self.params_json, f)
f.close()
def getDirs(self):
"""Return directories of difference pictures"""
return ([self.diff1_p, self.diff2_p], self.params,
self.contour) if self.done else (["", ""], "", "")
class GetFrames:
def __init__(self, root):
self.root = root
self.img_dir = ""
self.perSec = -1
self.params_json = ""
self.diff_dirs = ["", ""]
self.track_json = ""
self.contour_json = ""
self.window = tk.Toplevel(self.root)
self.window.geometry("350x200")
self.l0 = tk.Label(
self.window,
text="Select the video to process into frames,\n"
"entering the number of frames per second to extract\n"
"(default 0 means extract all frames available)\n"
"-or- select the directory containing frames:")
self.e0 = tk.Entry(self.window, width=4)
self.e0.insert(0, "0")
self.b0 = tk.Button(self.window,
text="Select video",
command=self.getFrames)
self.l1 = tk.Label(self.window, text="-or-")
self.b1 = tk.Button(self.window,
text="Select folder",
command=self.getFolder)
self.prog = Progress(self.window)
self.l0.pack()
self.e0.pack()
self.b0.pack()
self.l1.pack()
self.b1.pack()
def getFrames(self):
f = filedialog.askopenfilename()
if isinstance(f, str) and self.e0.get().isdigit():
self.img_dir = f
video = cv2.VideoCapture(f)
progress, frame = video.read()
(i, frameCount) = (0, 0)
fps = video.get(cv2.CAP_PROP_FPS)
self.perSec = min(int(self.e0.get()), fps)
ratio = int(fps // int(self.perSec)) if self.perSec else 1
length_of_vid = int(video.get(cv2.CAP_PROP_FRAME_COUNT) // ratio)
self.l0.config(
text=
f"~{length_of_vid} frames to convert for video. Video fps: {fps}. "
f"\nRatio of frames extracted per total: {ratio}.")
self.e0.pack_forget()
self.b0.pack_forget()
self.l1.pack_forget()
self.b1.pack_forget()
self.prog.pack()
self.window.update()
self.img_dir = f[:f.rfind(".")] + " sequence"
makedir(self.img_dir)
if len(str(length_of_vid)) < len(str(length_of_vid + 5)):
length_of_vid = length_of_vid + 5
form = "0>" + str(len(str(length_of_vid)))
self.prog.config(length=length_of_vid / 10)
while progress:
if i % ratio == 0:
cv2.imwrite(
f"{self.img_dir}/image{format(frameCount, form)}.jpg",
cv2.UMat(frame))
frameCount += 1
i += 1
progress, frame = video.read()
if i % (ratio * 10) == 0:
self.prog.increase()
video.release()
self.params_json = self.img_dir + params
if os.path.exists(self.params_json):
f = open(self.params_json)
temp = json.load(f)
f.close()
else:
temp = params_json
temp["duration"] = ratio / fps
f = open(self.params_json, "w")
json.dump(temp, f)
f.close()
self.exit()
else:
messagebox.showerror(title="Bad ratio",
message="Please type a non-negative integer "
"for the frame ratio entry box.")
def getFolder(self):
self.img_dir = filedialog.askdirectory()
if self.img_dir:
self.params_json = self.img_dir + params
self.diff_dirs = [self.img_dir + diff1, self.img_dir + diff2]
self.track_json = self.img_dir + track
self.contour_json = self.img_dir + contour
if not os.path.exists(self.params_json):
self.params_json = ""
if not (os.path.exists(self.diff_dirs[0]) and os.path.exists(
self.diff_dirs[1]) and len(os.listdir(self.diff_dirs[0]))
and len(os.listdir(self.diff_dirs[1]))):
self.diff_dirs = ["", ""]
if not os.path.exists(self.contour_json):
self.contour_json = ""
if not os.path.exists(self.track_json):
self.track_json = ""
self.exit()
def exit(self):
self.window.destroy()
def getFiles(self):
return self.img_dir, self.params_json, self.diff_dirs, self.track_json, self.contour_json
parser.add_option('-i', '--init', help="init the progress file",action="store_true",default=False)
parser.add_option('-e', '--error', help="re-examine the error pages",action="store_true",default=False)
parser.add_option('-s', '--sequence', help="examine progress to find unvisited profiles",action="store_true",default=False)
options, args = parser.parse_args()
session = Session()
if not session.doLogin():
sys.stderr.write("Something went wrong, still not connected\n")
if options.init:
sys.stderr.write("Init Progress File\n")
if options.profile == None and options.group == None:
sys.stderr.write("\tMust select profile or group to init from\n")
sys.exit(0)
progress = Progress()
progress.initPending(options.profile,options.group)
progress.printProgress()
crawler = Crawler(session,progress)
sys.stderr.write("Starting Crawler\n")
crawler.doTick()
sys.stderr.write("Ending Crawler\n")
progress.setExit()
progress.saveProgress()
sys.exit(0)
elif options.rebuild:
sys.stderr.write("Rebuilding Progress File\n")
Progress(True)
progress.printProgress()
sys.exit(0)
elif options.sequence:
class RealEstateSettings:
def __init__(self, inputdata, outputdata):
dao = DAOPsql('furman')
self.geo = GeoSearch(dao)
self.error_log = ErrorLog(self.__class__.__name__)
self.progress = Progress()
self.input = inputdata
self.output = outputdata
def fix_acris(self):
tuples = self.preprocess()
real_estates = []
while tuples:
try:
t = tuples.pop(0)
bbl = Normalizer.set_bbl(t[0], t[1], t[2])
address = t[3]+" "+t[4]
address = Normalizer.set_address(address, bbl)
date = Normalizer.set_str_to_epoch(t[5])
price = t[6]
real_estates.append((bbl, address, date, price))
except ValueError:
self.error_log.open()
self.error_log.write(t[1]+", "+str(t[0]))
self.error_log.close()
except KeyboardInterrupt:
print ""
print "Stopped"
CsvManager.append_geo_codes(real_estates, self.output)
CsvManager.append_geo_codes(real_estates, self.output)
def preprocess(self):
tuples = CsvManager.read(self.input)
num = CsvManager.read_progress()
print num
if num == 0:
CsvManager.write_geo_codes([], self.output)
CsvManager.write_progress('0')
self.progress.set_size(len(tuples))
self.progress.update_progress(num)
Normalizer.set_tuple(num, tuples)
return tuples
def build_geocodings(self):
nominatim = NominatimGeocode(self.progress, self.error_log, self.geo)
google = GoogleGeocode(self.progress, self.error_log, self.geo)
opencage = OpenCageGeocode(self.progress, self.error_log, self.geo)
bing = BingGeocode(self.progress, self.error_log, self.geo)
tiger = TIGERGeocode(self.progress, self.error_log, self.geo)
return nominatim, google, opencage, bing, tiger
def search_lat_long(self):
tuples = self.preprocess()
count = 1
nominatim, google, opencage, bing, tiger = self.build_geocodings()
while tuples:
t = tuples.pop(0)
status, found = self.geocode_process(t, nominatim)
if not found:
if status == -1:
status, found = self.geocode_process(t, bing)
if not found and status == -1:
self.geocode_process(t, tiger)
elif status == -2:
i = 1
while i < 3:
print "Waiting 45' for the "+Normalizer.set_order(str(i))+" time"
time.sleep(2700)
status, found = self.geocode_process(t, nominatim)
if found:
continue
elif status == -2:
i += 1
elif status == -3:
return
if count % 100 == 0:
for i in range(3):
t = tuples.pop(0)
status, found = self.geocode_process(t, google)
time.sleep(3)
if not found:
self.geocode_process(t, opencage)
time.sleep(3)
else:
t = tuples.pop(0)
self.geocode_process(t, opencage)
time.sleep(3)
count += 1
def geocode_process(self, t, geocode):
re, num = geocode.get_coordinates(t)
if num:
CsvManager.append_geo_codes([re], self.output)
self.progress.update_progress(num)
else:
val = CsvManager.read_progress()
self.progress.update_progress(val+1)
return re, num
except Exception, e:
raise ExtensionRunError( \
"Error creating repository %s. Exception: %s" % \
(repo.get_uri(), str(e)))
self.__prepare_table(connection)
fp = FilePaths(db)
read_cursor.execute(statement("""select COUNT(*)
from patches p, scmlog s
where p.commit_id = s.id and
s.repository_id = ? and
p.patch is not NULL""",
db.place_holder), (repo_id,))
nr_records = read_cursor.fetchone()[0]
progress = Progress("[Extension Hunks]", nr_records)
patches = self.get_patches(repo, path or repo.get_uri(), repo_id, db,
read_cursor)
for commit_id, file_id, patch_content, rev in patches:
for hunk in self.get_commit_data(patch_content):
insert = """insert into hunks(file_id, commit_id,
old_start_line, old_end_line, new_start_line,
new_end_line)
values(?,?,?,?,?,?)"""
execute_statement(statement(insert, db.place_holder),
(file_id, commit_id,
hunk.old_start_line,
hunk.old_end_line,
def fetch_progress(po_id, connection):
cursor = connection.cursor()
cursor.execute(
"SELECT prepped, thrown, debatted, trimmed, assembled, polished FROM public.purchaseorders WHERE id = %s"
% po_id)
return Progress(cursor.fetchone())
db.place_holder))
id = cursor.fetchone()[0]
if id is not None:
DBCommitLines.id_counter = id + 1
commits = self.__get_commits_lines_for_repository(repo_id, cursor)
except Exception, e:
raise ExtensionRunError(str(e))
counter = create_line_counter_for_repository(repo, uri)
cursor.execute(
statement(
"""SELECT id, rev, composed_rev from scmlog
where repository_id = ?""", db.place_holder), (repo_id, ))
progress = Progress("[Extension CommitsLOC]", cursor.rowcount)
write_cursor = cnn.cursor()
rs = cursor.fetchmany()
while rs:
commit_list = []
for commit_id, revision, composed_rev in rs:
if commit_id in commits:
continue
if composed_rev:
rev = revision.split("|")[0]
else:
rev = revision
def Prog(self):
self.prog = Progress(self)
self.prog.show()
self.close()
try:
self.__create_table(connection)
except TableAlreadyExists:
pass
except Exception, e:
raise ExtensionRunError(str(e))
cursor.execute(
statement(
"""select COUNT(*)
from patches p, scmlog s
where p.commit_id = s.id and
s.repository_id = ? and
p.patch is not NULL""", db.place_holder), (repo_id, ))
nr_records = cursor.fetchone()[0]
progress = Progress("[Extension PatchesLOC]", nr_records)
patches = self.get_patches(repo, path or repo.get_uri(), repo_id, db,
cursor)
for commit_id, file_id, patch_content, rev in patches:
(added, removed) = self.count_lines(patch_content)
insert = """insert into patch_lines(file_id, commit_id,
added, removed)
values(?,?,?,?)"""
execute_statement(statement(insert, db.place_holder),
(file_id, commit_id, added, removed),
cursor,
db,
"Couldn't insert patch, dup record?",
exception=ExtensionRunError)
class ScreamingMercury(QWidget, Ui_ScreamingMercury):
def __init__(self, delegate, parent=None):
QWidget.__init__(self, parent)
self.setupUi(self)
self.delegate = delegate
self.small_screwdriver = DrawBinsWidget()
self.work_layout.insertWidget(0, self.small_screwdriver)
self.progress_window = Progress()
self.settings_window = Settings()
self.bin_packing_thread = BinPackingThread()
self.addDirectory.clicked.connect(self.onAddDirectory)
self.removeImage.clicked.connect(self.onRemoveImages)
self.startPushButton.clicked.connect(self.onStart)
self.methodTabWidget.currentChanged.connect(self.bin_packing_thread.setMethod)
self.binSizeComboBox.currentIndexChanged.connect(self.bin_packing_thread.setBinSize)
self.clearPushButton.clicked.connect(self.delegate.on_remove_all_images)
self.settingsPushButton.clicked.connect(self.settings_window.show)
# self.images_changed.connect(self.updateImages)
self.bin_packing_thread.bin_packing_available.connect(self.startPushButton.setEnabled)
self.bin_packing_thread.on_end.connect(self.startPushButton.setEnabled)
self.bin_packing_thread.on_end.connect(self.progress_window.setHidden)
self.bin_packing_thread.prepare_progress_signal.connect(self.progress_window.prepareProgressBar.setValue)
self.bin_packing_thread.packing_progress_signal.connect(self.progress_window.binPackingProgressBar.setValue)
self.bin_packing_thread.saving_progress_signal.connect(self.progress_window.savingProgressBar.setValue)
self.startPushButton.setEnabled(self.bin_packing_thread.binPackingAvailable())
self.directory = None
self.images = []
self.restoreSettings()
if self.directory:
self.harvestDirectory(self.directory)
self.bin_packing_thread.update_bins.connect(self.small_screwdriver.redrawBins)
self.outputFormatComboBox.currentIndexChanged.connect(BinPackingThread.set_exporter)
BinPackingThread.set_exporter(self.outputFormatComboBox.currentIndex())
def restoreSettings(self):
# Настройки
self.settings = QSettings(QSettings.IniFormat, QSettings.UserScope, COMPANY, APPNAME)
print self.settings.fileName()
# Востанавливаем ...
# ... геометрию окна
self.restoreGeometry(self.settings.value(SETTINGS_GEOMETRY))
self.splitter.restoreState(self.settings.value(SETTINGS_SPLITTER1))
self.splitter_2.restoreState(self.settings.value(SETTINGS_SPLITTER2))
# ... размер контейнера
self.binSizeComboBox.setCurrentIndex(int(self.settings.value(SETTINGS_SIZE, defaultValue=0)))
# ... метод упаковки
self.methodTabWidget.setCurrentIndex(int(self.settings.value(SETTINGS_METHOD, defaultValue=0)))
# ... вариант упоковки от лучшего варианта или от худшего
self.firstFitShelfVariantComboBox.setCurrentIndex(
int(self.settings.value(SETTINGS_FIRST_FIT_VARIANT, defaultValue=0))
)
# ... эвристику упаковки
self.firstFitShelfHeuristicComboBox.setCurrentIndex(
int(self.settings.value(SETTINGS_FIRST_FIT_HEURISTIC, defaultValue=0))
)
# ... вариант для гильотины
self.guillotineVariantComboBox.setCurrentIndex(int(self.settings.value(SETTINGS_VARIANT, defaultValue=0)))
# ... эвристику для гильотины
self.guillotineHeuristicComboBox.setCurrentIndex(int(self.settings.value(SETTINGS_HEURISTIC, defaultValue=0)))
# ... правило разделения гильотиной
self.splitComboBox.setCurrentIndex(int(self.settings.value(SETTINGS_SPLIT_RULE, defaultValue=0)))
# ... масштаб отрисовки
self.small_screwdriver.scale = float(self.settings.value(SETTINGS_DRAW_SCALE, defaultValue=1.0))
self.outputFormatComboBox.setCurrentIndex(int(self.settings.value(SETTINGS_OUTPUT_FORMAT, defaultValue=0)))
self.directory = self.settings.value(SETTINGS_DIRECTORY, defaultValue=None)
self.progress_window.restoreGeometry(self.settings.value(SETTINGS_PROGRESS_WINDOW_GEOMETRY))
def onAddDirectory(self):
directory = QFileDialog.getExistingDirectory(caption="Select image directory", dir=self.directory)
if directory != u"":
self.directory = directory
self.harvestDirectory(self.directory)
def resetDirectory(self):
self.directory = None
self.images = []
self.bin_packing_thread.setDirectory(self.directory)
self.bin_packing_thread.setImages(self.images)
self.updateImages(self.images)
def harvestDirectory(self, directory):
folder = QDir(path=directory)
folder.setNameFilters(["*.png"])
folder.setFilter(QDir.Files or QDir.NoDotAndDotDot)
dit = QDirIterator(folder, flags=QDirIterator.Subdirectories, filters=QDir.Files)
while dit.hasNext():
im = folder.relativeFilePath(dit.next())
if not re.search("atlas", im):
self.images.append(im)
self.bin_packing_thread.setDirectory(self.directory)
self.bin_packing_thread.setImages(self.images)
self.updateImages(self.images)
def onRemoveImages(self):
row = self.imageList.currentRow()
del self.images[row]
self.updateImages(self.images)
self.imageList.setCurrentRow(row)
def onStart(self):
self.startPushButton.setEnabled(False)
self.bin_packing_thread.start()
self.progress_window.show()
def updateImages(self, images):
self.imageList.clear()
self.imageList.addItems(images)
def keyPressEvent(self, e):
if e.key() == Qt.Key_Escape:
self.close()
def closeEvent(self, e):
self.progress_window.close()
self.settings.setValue(SETTINGS_GEOMETRY, self.saveGeometry())
self.settings.setValue(SETTINGS_SPLITTER1, self.splitter.saveState())
self.settings.setValue(SETTINGS_SPLITTER2, self.splitter_2.saveState())
self.settings.setValue(SETTINGS_SIZE, self.binSizeComboBox.currentIndex())
self.settings.setValue(SETTINGS_METHOD, self.methodTabWidget.currentIndex())
self.settings.setValue(SETTINGS_FIRST_FIT_VARIANT, self.firstFitShelfVariantComboBox.currentIndex())
self.settings.setValue(SETTINGS_FIRST_FIT_HEURISTIC, self.firstFitShelfHeuristicComboBox.currentIndex())
self.settings.setValue(SETTINGS_VARIANT, self.guillotineVariantComboBox.currentIndex())
self.settings.setValue(SETTINGS_HEURISTIC, self.guillotineHeuristicComboBox.currentIndex())
self.settings.setValue(SETTINGS_SPLIT_RULE, self.splitComboBox.currentIndex())
self.settings.setValue(SETTINGS_DRAW_SCALE, self.small_screwdriver.scale)
self.settings.setValue(SETTINGS_OUTPUT_FORMAT, self.outputFormatComboBox.currentIndex())
self.settings.setValue(SETTINGS_DIRECTORY, self.directory)
# Progress Window
self.settings.setValue(SETTINGS_PROGRESS_WINDOW_GEOMETRY, self.progress_window.saveGeometry())
