def generate_distance_matrix(pickup_grid, delivery_grid, from_file=None):
'''
distance_matrix[job1id].keys() is all the neighbors of job1
distance_matrix[job1id][job2id] = total distance doing end(job1) -> start(job2) -> end(job2)
'''
if from_file:
return json.load(open(from_file))
distance_matrix = collections.defaultdict(dict)
print 'Building matrix...'
progress, total = 0, sum(len(v) for v in delivery_grid.values())
print total
for (start_lat, start_lng) in delivery_grid:
neighbors = util.neighbors(pickup_grid, start_lat, start_lng, k=2)
for job1 in delivery_grid[(start_lat, start_lng)]:
util.print_progress(progress, total)
for job2 in neighbors:
if job2['pickupDate'] < job1['pickupDate']: continue
d1 = util.interjob_distance(job1, job2)
d2 = util.intrajob_distance(job2)
distance_matrix[job1['_id']['$oid']][job2['_id']['$oid']] = d1 + d2
progress += 1
with open(os.path.join('project_data', 'distance_matrix.json'), 'w') as out:
json.dump(distance_matrix, out) # Dump it so that we don't have to run this entire procedure again
return distance_matrix
def download_asset(temp_dir: str, url: str, filename: str):
"""Downloads a file in 512 byte chunks from a GitHub octet-stream.
:param temp_dir: Temporary directory to be used
:type temp_dir: str
:param url: URL to the octet-stream API call
:type url: str
:param filename: Name of the downloaded file
:type filename: str
"""
logging.info("Sending asset download request...")
response = requests.send_request(url, "application/octet-stream")
response_size = int(response.headers['content-length'])
logging.debug(f"Started downloading {response_size} bytes...")
with open(os.path.join(temp_dir, filename), "wb") as file:
bytes_read = 0
chunk_size = 512
for chunk in response.iter_content(chunk_size):
file.write(chunk)
bytes_read += chunk_size
util.print_progress(bytes_read, response_size, suffix="Downloaded")
logging.info(f"Successfully finished downloading {filename}")
return True
def migrate_users(old_db, new_db):
old_user_coll = old_db["user"]
new_user_coll = new_db["user"]
old_users = old_user_coll.find()
user_migrates = {}
total_user_count = old_user_coll.count_documents({})
processed_count = 0
migrated_count = 0
skipped_count = 0
failed_usernames = []
for old_user in old_users:
result = migrate_user(old_user, new_user_coll)
if "failed" in result:
failed_usernames.append(result["username"])
else:
new_user_id = result["_id"]
user_migrates[old_user["_id"]] = new_user_id
if result["existed"]:
skipped_count += 1
else:
migrated_count += 1
processed_count += 1
print_progress(processed_count,
total_user_count,
prefix="Migrating users...")
failed_count = len(failed_usernames)
print(
f"Total: {total_user_count}, migrated: {migrated_count}, skipped: {skipped_count}, failed: {failed_count}"
)
if failed_count > 0:
print(f"Failed usernames: {failed_usernames}")
return user_migrates
def load_invoices_and_map_labels(self,
data_path: str,
autoload=False,
verbose: bool = False):
# This tuple represents the number of pages to do OCR for for each invoice. Eg. (2,1) represents do OCR for the first 2 pages, and for the last page
RANGE_OF_PAGES_FOR_OCR = (2, 2)
invoices = []
pdf_list = list(
filter(lambda file_name: file_name.endswith(".pdf"),
os.listdir(data_path)))
for index, filename in enumerate(pdf_list):
invoice = Invoice(data_path + "/" + filename)
if autoload:
loaded = invoice.load_data()
if loaded:
invoices.append(invoice)
else:
raise Exception("Autoloading of invoice",
invoice.readable_name,
"failed. Check if the savefile exists")
else:
# First check if json tags are present. If they aren't, skip this pdf
if not os.path.exists(data_path + "/" + filename[:-4] +
".json"):
print(
"Warning: json tags for",
filename,
"does not exist. Check if they are in the same folder. Skipping this pdf",
)
continue
# Next, do OCR for the relevant pages in the invoice
if verbose:
print("\n Processing:", invoice.readable_name)
if invoice.length() < sum(RANGE_OF_PAGES_FOR_OCR):
for page in invoice.pages:
page.do_OCR(verbose=verbose)
else:
for page in invoice.pages[:RANGE_OF_PAGES_FOR_OCR[
0]]: # Initial pages
page.do_OCR(verbose=verbose)
for page in invoice.pages[
-RANGE_OF_PAGES_FOR_OCR[1]:]: #Last pages
page.do_OCR(verbose=verbose)
# Try mapping labels
invoice.map_labels(verbose=verbose)
invoices.append(invoice)
invoice.save_data()
if verbose:
print_progress(index + 1, len(pdf_list), "Loading invoices ")
return invoices
def _connect(self):
"""spawn the interactive subprocess."""
if not (hasattr(self, 'cmdline') and self.cmdline):
# No cmd to run, return a class instance without subprocess process.
return None
# open the process, as a subprocess.Popen object.
self.print_input('Starting interactive-process %s: %s\n' % (self.name, self.cmdline))
cmd_list = self.cmdline if self.use_shell else shlex.split(self.cmdline)
for attempt in xrange(self._init_retry + 1):
if self.use_pty_stdin or self.use_pty_stdout:
# Use a PTY pseudo terminal, for program that does tcgetattr, or other cases.
# with util.SudoPrivilege(): #FIXME: openpty may get 'Out of devices' error w/o sudo
master, slave = pty.openpty()
p = subprocess.Popen(cmd_list, bufsize=0,
stdin=slave if self.use_pty_stdin else subprocess.PIPE,
stdout=slave if self.use_pty_stdout else subprocess.PIPE,
stderr=subprocess.PIPE, shell=self.use_shell,
universal_newlines=True)
self.stdin = os.fdopen(master, 'w', 0) if self.use_pty_stdin else p.stdin
self.stdout = os.fdopen(master, 'rU', 0) if self.use_pty_stdout else p.stdout
self.stderr = p.stderr
if not p:
self.print_warn('fail to open %s process, "%s".' % (self.name, self.cmdline))
# change the pipe to non-blocking
util.tty_nonblocking(self.stdout)
util.tty_nonblocking(self.stderr)
# disable echo for cleaner output
if self._disable_echo:
util.term_set_echo(self.stdin.fileno(), enable=False)
self.process = p
self.peek()
if self.is_alive():
# connected
break
util.print_progress('Will retry "%s" in 1 sec, %s attempt ...' %
(self.cmdline, attempt + 1), color=['cyan'])
time.sleep(1)
self.print_input('\n')
if not self.is_alive():
self.print_error('fail to connect to %s, "%s", with %d retries.'
% (self.name, self.cmdline, self._init_retry))
return None
if self._init_flush:
# set continuous_output to reuse previous flushed output
self.flush(expect='', hide_output=self.hide_output, continuous_output=True)
self._had_connect = True
return self.process
def migrate_mangas(old_db, new_db):
old_manga_coll = old_db["manga"]
new_manga_coll = new_db["manga"]
old_mangas = old_manga_coll.find()
total_manga_count = old_manga_coll.count_documents({})
migrated_count = 0
processed_count = 0
skipped_count = 0
failed_dmk_ids = []
manga_migrates = {}
# Generate the batches
batch_size = 50
batches = []
count = 0
for old_manga in old_mangas:
if count == 0:
batches.append([])
batches[-1].append(old_manga)
count += 1
if count == batch_size:
count = 0
# Fetch in batches
for batch in batches:
# Generate the threads
threads = [] # List[Tuple[Manga, Thread]]
for old_manga in batch:
thread = migrate_manga(old_manga, new_manga_coll)
threads.append((old_manga, thread))
# print_progress(len(threads), total_manga_count, prefix="Spawning threads...")
# Join all the threads and get the result
for (old_manga, thread) in threads:
result = thread.wait()
processed_count += 1
if result["failed"]:
failed_dmk_ids.append(old_manga["dmk_id"])
else:
if result["existed"]:
skipped_count += 1
else:
migrated_count += 1
manga_migrates[old_manga["_id"]] = result
print_progress(processed_count, total_manga_count, prefix="Migrating mangas...")
failed_count = len(failed_dmk_ids)
print(f"Total: {total_manga_count}, Completed: {migrated_count}, Skipped: {skipped_count}, Failed: {failed_count}")
if failed_count > 0:
print(f"Failed dmk ids: {failed_dmk_ids}")
return manga_migrates
def cmd_poll(self, cmd, pattern, reverse=False, sum_value=None, max_times=10, interval=0.1,
initial_delay=0, verbose=1, *args, **kwargs):
"""Execute a command multiple times until a specified pattern is in the command output.
E.g. waiting for a prcoess completing some time consuming task by polling a command.
- cmd: command to be executed.
- pattern: a regex pattern, once it is found in the command output, polling is completed.
- reverse: reverse the logic, once the pattern is not longer found, polling is completed.
- sum_value: if specified, poll till the sum of multiple pattern instances matches the value.
Note, using both reverse and sum_value is not supported.
- max_times: specify the maximum times the command should be executed to wait for the pattern.
- interval: time interval between command execution, specified in fraction number of seconds.
- initial_delay: time to be delayed before starting the polling.
- verbose: 0: no log, 1: show output of last poll, 2: show output from all polling iterations.
- \*args, \*\*kwargs: optional parameters for self.cmd()
- return: (output, stderr_output, result), outputs of the last execution, and whether the polling
succeeds in getting expected result.
"""
o = e = ''
i = 0
for i in xrange(max_times + 1):
if not self.is_alive():
self.print_error('\n%s not alive, cmd_poll "%s" returned, polled %d times' %
(self.name, cmd, i))
return o, e, None
time.sleep(initial_delay if i == 0 else interval)
o, e, r = self.cmd_search(cmd, pattern=pattern, reverse=reverse, sum_value=sum_value,
verbose=(verbose == 2), *args, **kwargs)
if r:
break
progress = ('cmd_poll: time spent %.2f seconds, (delay %s, interval %s, %s times), result %s ...' %
(i * interval + initial_delay, initial_delay, interval, i, r))
if verbose == 2:
self.print_input('%s\n' % progress)
elif verbose == 1:
util.print_progress(progress, color=['cyan'])
else:
# did not get the expected polling result
r = False
util.print_progress('')
if verbose:
self.print_input('%s cmd_poll "%s": waited %.2f seconds, result %s\n'
% (self.name, cmd, i * interval + initial_delay, r))
if not r:
self.print_error('%s cmd_poll failed %sto get "%s", waited %.2f seconds' %
(self.name, 'NOT ' if reverse else '', pattern, i * interval + initial_delay))
if verbose == 1 or (verbose == 0 and not r):
self.print_output('\n%s\n' % o)
self.print_stderr('%s\n' % e)
return o, e, r
def evaluate_network(network, target, target_host, number):
# connect to remote device
tracker = tvm.rpc.connect_tracker(args.host, args.port)
remote = tracker.request(args.rpc_key)
print_progress(network)
net, params, input_shape, output_shape = get_network(network, batch_size=1)
print_progress("%-20s building..." % network)
with nnvm.compiler.build_config(opt_level=3):
graph, lib, params = nnvm.compiler.build(net,
target=target,
target_host=target_host,
shape={'data': input_shape},
params=params,
dtype=dtype)
tmp = tempdir()
if 'android' in str(target):
from tvm.contrib import ndk
filename = "%s.so" % network
lib.export_library(tmp.relpath(filename), ndk.create_shared)
else:
filename = "%s.tar" % network
lib.export_library(tmp.relpath(filename))
# upload library and params
print_progress("%-20s uploading..." % network)
ctx = remote.context(str(target), 0)
remote.upload(tmp.relpath(filename))
rparams = {k: tvm.nd.array(v, ctx) for k, v in params.items()}
rlib = remote.load_module(filename)
module = runtime.create(graph, rlib, ctx)
data_tvm = tvm.nd.array(
(np.random.uniform(size=input_shape)).astype(dtype))
module.set_input('data', data_tvm)
module.set_input(**rparams)
del rparams
# evaluate
print_progress("%-20s evaluating..." % network)
ftimer = module.module.time_evaluator("run",
ctx,
number=args.number,
repeat=3)
prof_res = np.array(
ftimer().results) * 1000 # multiply 1000 for converting to millisecond
print(
"%-20s %-19s (%s)" %
(network, "%.2f ms" % np.mean(prof_res), "%.2f ms" % np.std(prof_res)))
def geometric_voting_abs(boxes, km_pp, r_tol, d_tol, wmax, verb=False):
global radius_candidates, rtol, dtol, centers, vote_db, kmpp, w_max, relative_search, verbose
relative_search = False
vote_db = {}
radius_candidates = {}
dtol = d_tol
rtol = r_tol
w_max = wmax
kmpp = km_pp
n = boxes.shape[0]
radii = np.zeros(n)
center_points = np.zeros((n, 2))
# calculate all radii and center points
for i, box in enumerate(boxes):
radii[i] = util.calculate_radius(box) * kmpp
center_points[i] = util.calculate_center(box)
# get sorting vector
sort = radii.argsort()[::-1]
# sort dat shit
radii = radii[sort]
center_points = center_points[sort]
center_points *= kmpp
centers = center_points
# get the radius candidates
for c, r in enumerate(radii):
radius_candidates[c] = pdb.get_craters_by_real_radius(r, r_tol)
tuples = list(combos(np.arange(n), 2))
t = len(tuples)
intersect = False
for count, (ci, cj) in enumerate(tuples):
util.print_progress(count, t, 'Pairs loop', 'complete')
ci_candidates, cj_candidates = search_pair(ci, cj, False)
vote_db = add_votes_to_db(vote_db, ci, ci_candidates)
vote_db = add_votes_to_db(vote_db, cj, cj_candidates)
if not intersect:
vote_db = add_votes_to_db(vote_db, ci, radius_candidates[ci])
vote_db = add_votes_to_db(vote_db, cj, radius_candidates[cj])
# check_db()
return do_final_validation()
def evaluate_network(network, target, target_host, dtype, repeat):
# connect to remote device
tracker = tvm.rpc.connect_tracker(args.host, args.port)
remote = tracker.request(args.rpc_key)
print_progress(network)
net, params, input_shape, output_shape = get_network(network,
batch_size=1,
dtype=dtype)
print_progress("%-20s building..." % network)
with tvm.transform.PassContext(opt_level=3):
lib = relay.build(net,
target=target,
target_host=target_host,
params=params)
tmp = tempdir()
if "android" in str(target) or "android" in str(target_host):
from tvm.contrib import ndk
filename = "%s.so" % network
lib.export_library(tmp.relpath(filename), ndk.create_shared)
else:
filename = "%s.tar" % network
lib.export_library(tmp.relpath(filename))
# upload library and params
print_progress("%-20s uploading..." % network)
dev = remote.device(str(target), 0)
remote.upload(tmp.relpath(filename))
rlib = remote.load_module(filename)
module = runtime.GraphModule(rlib["default"](dev))
data_tvm = tvm.nd.array(
(np.random.uniform(size=input_shape)).astype(dtype))
module.set_input("data", data_tvm)
# evaluate
print_progress("%-20s evaluating..." % network)
ftimer = module.module.time_evaluator("run", dev, number=1, repeat=repeat)
prof_res = np.array(
ftimer().results) * 1000 # multiply 1000 for converting to millisecond
print(
"%-20s %-19s (%s)" %
(network, "%.2f ms" % np.mean(prof_res), "%.2f ms" % np.std(prof_res)))
def evaluate_network(network, target, target_host, dtype, repeat):
# connect to remote device
tracker = tvm.rpc.connect_tracker(args.host, args.port)
remote = tracker.request(args.rpc_key)
print_progress(network)
net, params, input_shape, output_shape = get_network(network, batch_size=1, dtype=dtype)
print_progress("%-20s building..." % network)
with nnvm.compiler.build_config(opt_level=3):
graph, lib, params = nnvm.compiler.build(
net, target=target, target_host=target_host,
shape={'data': input_shape}, params=params, dtype=dtype)
tmp = tempdir()
if 'android' in str(target) or 'android' in str(target_host):
from tvm.contrib import ndk
filename = "%s.so" % network
lib.export_library(tmp.relpath(filename), ndk.create_shared)
else:
filename = "%s.tar" % network
lib.export_library(tmp.relpath(filename))
# upload library and params
print_progress("%-20s uploading..." % network)
ctx = remote.context(str(target), 0)
remote.upload(tmp.relpath(filename))
rlib = remote.load_module(filename)
module = runtime.create(graph, rlib, ctx)
data_tvm = tvm.nd.array((np.random.uniform(size=input_shape)).astype(dtype))
module.set_input('data', data_tvm)
module.set_input(**params)
# evaluate
print_progress("%-20s evaluating..." % network)
ftimer = module.module.time_evaluator("run", ctx, number=1, repeat=repeat)
prof_res = np.array(ftimer().results) * 1000 # multiply 1000 for converting to millisecond
print("%-20s %-19s (%s)" % (network, "%.2f ms" % np.mean(prof_res), "%.2f ms" % np.std(prof_res)))
def migrate_follows(old_db, new_db, user_migrates, manga_migrates):
"""
user_migrates: { [old_user_id]: [new_user_id] }
manga_migrates: { [old_manga_id]: { "_id": [new_manga_id], "dmk_id": [dmk_id] } }
"""
old_follow_coll = old_db["follow"]
new_follow_coll = new_db["follow"]
old_follows = old_follow_coll.find()
total_follow_count = old_follow_coll.count_documents({})
migrated_count = 0
processed_count = 0
skipped_count = 0
failed_follows = []
for old_follow in old_follows:
old_user_id = old_follow["user_id"]
old_manga_id = old_follow["manga_id"]
try:
if old_user_id in user_migrates:
new_user_id = user_migrates[old_user_id]
if old_manga_id in manga_migrates:
manga_dmk_id = manga_migrates[old_manga_id]["dmk_id"]
# Check if the follow has already existed
new_follow = new_follow_coll.find_one({
"user_id":
ObjectId(new_user_id),
"manga_dmk_id":
manga_dmk_id
})
if new_follow:
skipped_count += 1
else:
# Is liked information
if "liked" in old_follow:
is_liked = old_follow["liked"]
else:
is_liked = False
if "start_date" in old_follow:
start_date_time = old_follow["start_date"]
else:
start_date_time = datetime.datetime.utcnow()
# Generate a new follow document
new_follow = {
"user_id": ObjectId(new_user_id),
"manga_dmk_id": manga_dmk_id,
"start_date_time": start_date_time,
"update_date_time": old_follow["update_date"],
"current_episode": old_follow["current_episode"],
"max_episode": old_follow["max_episode"],
"is_up_to_date": old_follow["up_to_date"],
"is_liked": is_liked
}
# Insert into the new collection
new_follow_coll.insert_one(new_follow)
migrated_count += 1
else:
failed_follows.append({
"old_user_id": old_user_id,
"old_manga_id": old_manga_id,
"reason": "Manga not found"
})
else:
failed_follows.append({
"old_user_id": old_user_id,
"old_manga_id": old_manga_id,
"reason": "User not found"
})
except Exception as err:
failed_follows.append({
"old_user_id": old_user_id,
"old_manga_id": old_manga_id,
"reason": str(err)
})
finally:
processed_count += 1
print_progress(processed_count,
total_follow_count,
prefix="Migrating follow...")
failed_count = len(failed_follows)
print(
f"Total: {total_follow_count}, migrated: {migrated_count}, skipped: {skipped_count}, failed: {failed_count}"
)
if failed_count > 0:
print(f"Failed follows: {failed_follows}")
def encode(encoder, threads, ffmpeg_path, encoder_path, job):
encoder_params = job.encoder_params
ffmpeg_params = job.ffmpeg_params
vfs = [f"select=gte(n\\,{job.start})"]
if "-vf" in ffmpeg_params:
idx = ffmpeg_params.index("-vf")
del ffmpeg_params[idx]
if idx + 1 < len(ffmpeg_params):
vfs.append(ffmpeg_params[idx + 1])
del ffmpeg_params[idx + 1]
vf = ",".join(vfs)
output_filename = f"tmp{job.segment}.ivf"
log_path = f"tmp{job.segment}.log"
ffmpeg = [
ffmpeg_path,
"-y",
"-loglevel",
"error",
"-i",
job.filename,
"-strict",
"-1",
"-pix_fmt",
"yuv420p10le",
"-vsync",
"0",
"-vf",
vf,
"-vframes",
job.frames,
]
ffmpeg.extend(ffmpeg_params)
ffmpeg.extend(["-f", "yuv4mpegpipe", "-"])
ffmpeg = [str(s) for s in ffmpeg]
aom = [
encoder_path,
"-",
"--ivf",
f"--fpf={log_path}",
f"--threads={threads}",
f"--passes={job.passes}",
"-o",
output_filename,
] + encoder_params
aom = [str(s) for s in aom]
if job.passes == 2:
pass1 = [a for a in aom if not a.startswith("--denoise-noise-level")]
passes = [
pass1 + ["--pass=1"],
aom + ["--pass=2"],
]
# if job.grain_table:
# if not job.has_grain:
# return False, None
# else:
# passes[-1].append(f"--film-grain-table={job.grain}")
total_frames = job.frames
ffmpeg_pipe = None
try:
for pass_n, cmd in enumerate(passes, start=1):
ffmpeg_pipe = subprocess.Popen(
ffmpeg,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
job.pipe = subprocess.Popen(
cmd,
stdin=ffmpeg_pipe.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
)
job.progress = (pass_n, 0)
job.update_status(
"{:.3s}".format(encoder),
"pass:"******"frame.*?\/([^ ]+?) ", line)
if match:
frames = int(match.group(1))
job.progress = (pass_n, frames)
job.update_progress()
job.update_status(
"{:.3s}".format(encoder),
"pass:"******"\n".join(output))
except JobStopped:
logging.info("Job stopped")
return None
except:
logging.error(traceback.format_exc())
return None
finally:
if ffmpeg_pipe:
ffmpeg_pipe.kill()
job.pipe.kill()
if os.path.isfile(log_path):
try:
os.remove(log_path)
except:
pass
return output_filename
def add_missing_columns(con, table_name, db_schema_str, site_url_visit_id_map):
col_to_replace = None
existing_columns = get_column_names_from_db_schema_str(
table_name, db_schema_str)
if "top_url" in existing_columns:
col_to_replace = "top_url"
elif "page_url" in existing_columns:
col_to_replace = "page_url"
# column names from the up to date DB schema
new_columns = get_column_names_from_create_query(TABLE_SCHEMAS[table_name])
if new_columns == existing_columns:
# print "No missing columns to add to", table_name
return False
print "Will add missing columns to %s: %s" % (
table_name, set(new_columns).difference(set(existing_columns)))
processed = 0
data_to_insert = []
num_rows = con.execute("SELECT MAX(id) FROM %s" % table_name).fetchone()[0]
# Copy the existing table to a temp table
tmp_table_name = "_%s_old" % table_name
con.execute("ALTER TABLE %s RENAME TO %s;" % (table_name, tmp_table_name))
# create table with the most recent schema
con.execute(TABLE_SCHEMAS[table_name])
# only keep the columns that also appear in the new table schema
common_columns = [
column for column in existing_columns if column in new_columns
]
t0 = time()
# replace top_url and page_url columns with visit_id
if col_to_replace:
print "Will replace %s with visit_id" % col_to_replace
assert "visit_id" not in existing_columns
# select from columns that are common to old and new table schemas
# col_to_replace is either top_url or page_url
# we use is to get the visit_id
cols_to_select = common_columns + [
col_to_replace,
]
cols_to_insert = common_columns + [
"visit_id",
]
stream_qry = "SELECT %s FROM %s " % (",".join(cols_to_select),
tmp_table_name)
# print "Will iterate over", stream_qry
insert_qry = "INSERT INTO %s (%s) VALUES (%s)" % (table_name, ",".join(
cols_to_insert), ",".join("?" * len(cols_to_insert)))
for row in con.execute(stream_qry):
try:
visit_id = site_url_visit_id_map[row[col_to_replace]]
except Exception:
print "Warning: Missing visit id", col_to_replace, row
continue
row = list(row)
row.pop() # remove col_to_replace, we don't need it anymore
row.append(visit_id) # add visit_id
data_to_insert.append(row)
# print "Will execute %s" % qry
# con.execute(qry, row)
processed += 1
if processed % 100000 == 0:
con.executemany(insert_qry, data_to_insert)
del data_to_insert[:]
print_progress(t0, processed, num_rows)
con.executemany(insert_qry, data_to_insert)
else:
# read from the temp table and write into the new table
stream_qry = "SELECT %s FROM %s " % (",".join(common_columns),
tmp_table_name)
# print "Will iterate over", stream_qry
insert_qry = "INSERT INTO %s (%s) VALUES (%s)" % (table_name, ",".join(
common_columns), ",".join("?" * len(common_columns)))
for row in con.execute(stream_qry):
data_to_insert.append(row)
# print "Will execute %s" % qry
# con.execute(insert_qry, row)
processed += 1
if processed % 100000 == 0:
con.executemany(insert_qry, data_to_insert)
del data_to_insert[:]
print_progress(t0, processed, num_rows)
con.executemany(insert_qry, data_to_insert)
t0 = time()
print "Will drop the temp table",
con.execute("DROP TABLE %s" % tmp_table_name)
print "(took", time() - t0, "s)"
con.commit()
return True
def grad_and_mass(rows, cols, mask=None, skip=None):
"""
Returns a gradient operator matrix G for a grid of dimensions
'rows' by 'cols',
a corresponding mass matrix M such that L = G.T*M*G is a
Laplacian operator matrix that is symmetric and normalized such that the
diagonal values of interior vertices equal 1,
and a skip matrix S such that the gradient entries of S*G*x are zero for
(i,j) such that skip[i,j] is False. If skip is None, all entries are
assumed to be True and S will be the identity matrix.
Optional parameter `mask` will result in a gradient operator that entirely
ignores (i,j) such that mask[i,j] is False.
In other words, `mask` should be True for every grid location
you care about (want a solution to via L = G.T*M*G).
`skip` should be True for every grid location you have a known good value
for.
Matrices returned are scipy.sparse matrices.
"""
print_progress(0)
assert rows > 0 and cols > 0
if mask is not None:
mask = asarray(mask, dtype=bool)
assert mask.shape == (rows, cols)
if skip is not None:
skip = asarray(skip, dtype=bool)
assert skip.shape == (rows, cols)
# The number of derivatives in the +row direction is cols * (rows - 1),
# because the bottom row doesn't have them.
num_Grow = cols * (rows - 1)
# The number of derivatives in the +col direction is rows * (cols - 1),
# because the right-most column doesn't have them.
num_Gcol = rows * (cols - 1)
# Gradient matrix
gOnes = numpy.ones(num_Grow + num_Gcol)
vals = numpy.append(-gOnes, gOnes)
del gOnes
gColRange = numpy.arange(rows * cols)
gColRange = gColRange[~(gColRange % cols == (cols - 1))]
colJ = numpy.concatenate([
numpy.arange(num_Grow), gColRange,
numpy.arange(cols, num_Grow + cols), gColRange + 1
])
del gColRange
# Skip matrix
if (skip is not None):
S_diag = numpy.append(skip[:-1] & skip[1:],
skip[:, :-1] & skip[:, 1:]).astype(int)
else:
S_diag = numpy.ones(num_Grow + num_Gcol)
# Mass diagonal matrix
if (mask is not None):
m = numpy.zeros((rows - 1, cols))
m[:, 1:][mask[:-1, :-1] & mask[1:, :-1]] += 0.125
m[:, :-1][mask[:-1, 1:] & mask[1:, 1:]] += 0.125
mass = m.flatten()
m = numpy.zeros((rows, cols - 1))
m[1:][mask[:-1, :-1] & mask[:-1, 1:]] += 0.125
m[:-1][mask[1:, :-1] & mask[1:, 1:]] += 0.125
mass = numpy.append(mass, m)
else:
m = numpy.hstack([
numpy.full((rows - 1, 1), 0.125),
numpy.full((rows - 1, cols - 2), 0.25),
numpy.full((rows - 1, 1), 0.125)
])
mass = m.flatten()
m = numpy.vstack([
numpy.full((1, cols - 1), 0.125),
numpy.full((rows - 2, cols - 1), 0.25),
numpy.full((1, cols - 1), 0.125)
])
mass = numpy.append(mass, m.flatten())
del m
output_row = num_Grow + num_Gcol
if (mask is not None):
keep_rows = numpy.append(mask[:-1] & mask[1:],
mask[:, :-1] & mask[:, 1:])
tiled_keep_rows = numpy.tile(keep_rows, 2)
vals = vals[tiled_keep_rows]
colJ = colJ[tiled_keep_rows]
S_diag = S_diag[keep_rows]
mass = mass[keep_rows]
output_row = numpy.count_nonzero(keep_rows)
# rowI is dependent on the number of output rows.
rowI = numpy.tile(numpy.arange(output_row), 2)
G = sparse.coo_matrix((vals, (rowI, colJ)),
shape=(output_row, rows * cols))
assert G.shape == (output_row, rows * cols)
M = coo_diag(mass)
assert M.shape == (output_row, output_row)
S = coo_diag(S_diag)
assert S.shape == (output_row, output_row)
print_progress(1.0)
print()
return G, M, S
if args.network is None:
networks = ['squeezenet_v1.1', 'mobilenet', 'resnet-18', 'vgg-16']
else:
networks = [args.network]
target = tvm.target.arm_cpu(model=args.device)
# connect to remote device
tracker = tvm.rpc.connect_tracker(args.host, args.port)
remote = tracker.request(args.rpc_key)
print("--------------------------------------------------")
print("%-20s %-20s" % ("Network Name", "Mean Inference Time (std dev)"))
print("--------------------------------------------------")
for network in networks:
print_progress(network)
net, params, input_shape, output_shape = get_network(network,
batch_size=1)
print_progress("%-20s building..." % network)
with nnvm.compiler.build_config(opt_level=2,
add_pass=['AlterOpLayout']):
graph, lib, params = nnvm.compiler.build(
net,
target=target,
shape={'data': input_shape},
params=params,
dtype=dtype)
tmp = tempdir()
if 'android' in str(target):
