def cmd(args):
_fs = Fs(args.host, args.db, args.user, args.password, args.bucket,
args.ssl, args.auth_db)
if exists(args.source):
local_folder, local_files = _build_local_file_list(args.source)
if len(local_files) > 0:
# build remote gridFs prefix (prepend a '/' if required)
remote_prefix = ''
if len(args.prefix) > 0:
remote_prefix = "/{0}".format(
args.prefix
) if not args.prefix.startswith('/') else args.prefix
for local_file in local_files:
local = local_folder + local_file
remote = remote_prefix + local_file
src_file = open(local, 'r')
spinner = Spinner()
sys.stdout.write(remote + ': ')
spinner.start()
f_id = _fs.upload(src_file, remote)
spinner.stop()
sys.stdout.write(str(f_id) + '\n')
else:
print "No files to upload."
else:
print 'local file/folder [{0}] does not exists'.format(args.source)
def __init__(self):
super.__init__
self.HOME_PATH = ""
self.main_qml = ""
self.control = ()
self.fileSys = Fs()
self._preprocesses()
def __init__(self, config):
self.config = config
self.twitter = Twython(config.TwitterConsumerKey,
config.TwitterConsumerSecret,
config.TwitterAccessToken,
config.TwitterAccessTokenSecret)
self.foursquare = Fs(config)
def cmd(args):
_fs = Fs(args.host, args.db, args.user, args.password, args.bucket,
args.ssl, args.auth_db)
files = _fs.find(args.filename)
if len(files) > 0:
if args.long_format:
_long_display(files)
else:
_display(files)
else:
print "no matching file found."
def __init__(self):
auth = dict(url=get_env("MONGO_URL", required=False),
host=get_env("MONGO_HOST", required=False),
port=get_env("MONGO_PORT", required=False),
db_name=get_env("MONGO_DBNAME"),
username=get_env("MONGO_INITDB_ROOT_USERNAME",
required=False),
password=get_env("MONGO_INITDB_ROOT_PASSWORD",
required=False))
self.target_collection_name = get_env("TARGET_COLLECTION_NAME")
self.target_collection = None
self.aux_collection_name = get_env("AUX_COLLECTION_NAME")
self.aux_collection = None
self.fs = Fs()
self.db = Db(auth)
def cmd(args):
_fs = Fs(args.host, args.db, args.user, args.password, args.bucket,
args.ssl, args.auth_db)
confirmation = args.confirmation
files = _fs.find(args.filename)
if len(files) > 0:
print "Found {0} files to remove".format(len(files))
confirmed = _confirm_removal() if confirmation else False
if not confirmation or confirmed:
for f in files:
spinner = Spinner()
print f.filename
spinner.start()
_fs.rm(f._id)
spinner.stop()
else:
print "no matching file found."
def handler(event, context):
config = KMS()
db = Dynamo(config)
foursquare = Fs(config)
twitter = Twitter(config)
client = RemoteCKAN('https://data.boston.gov')
viols = get_viols(client)
client.close()
for viol in viols:
if not db.query(viol['_id']):
count = db.count(viol['LICENSENO'])
url = format_url(viol)
text = format_msg(viol, count, url)
(lat, lon) = extract_geo(viol)
place = foursquare.place_search(name=viol['businessName'],
lat=lat,
lon=lon)
photo_url = None
if place:
photo_url = foursquare.random_photo_url(place)
twitter.tweet(text, photo_url, lat, lon)
db.save(viol['_id'], viol['LICENSENO'])
break
else:
print('Violation already known to Dynamo')
def cmd(args):
_fs = Fs(args.host, args.db, args.user, args.password, args.bucket,
args.ssl, args.auth_db)
gridfs_files = _fs.find(args.filename)
nb_of_files = len(gridfs_files)
if nb_of_files > 0:
print 'downloading ' + str(nb_of_files) + ' files:'
for gridfs_file in gridfs_files:
gridfs_filename = gridfs_file.filename
destination = args.destination + gridfs_filename
# check for any non-existing parent directories for local destination, and create them
destination_root = dirname(destination)
if not exists(destination_root):
makedirs(destination_root)
spinner = Spinner()
spinner.start()
dst_file = open(destination, 'wb')
_fs.download(gridfs_filename, dst_file)
spinner.stop()
print destination
else:
print "no matching file found."
class Twitter:
def __init__(self, config):
self.config = config
self.twitter = Twython(config.TwitterConsumerKey,
config.TwitterConsumerSecret,
config.TwitterAccessToken,
config.TwitterAccessTokenSecret)
self.foursquare = Fs(config)
def tweet(self, text, img, lat, lon):
if self.config.dev:
print(u'Tweeting to dev acct: {} @ ({}°, {}°)'.format(
text, lat, lon))
if img:
print("Will have image")
try:
if img:
img_obj = self.foursquare.photo_from_url(img)
img_response = self.twitter.upload_media(media=img_obj)
self.twitter.update_status(
status=text,
lat=lat,
long=lon,
display_coordinates=True,
media_ids=[img_response['media_id']])
else:
self.twitter.update_status(status=text,
lat=lat,
long=lon,
display_coordinates=True)
except TwythonError as e:
print(e)
class MusicApp():
def __init__(self):
super.__init__
self.HOME_PATH = ""
self.main_qml = ""
self.control = ()
self.fileSys = Fs()
self._preprocesses()
def _postprocesses(self):
"""
"""
self.control.app_running = False
self.fileSys.app_running = False
self.rebuildFiles()
self.cleanUpFiles()
def rebuildFiles(self):
# folder list
with open(self.HOME_PATH + "/" + 'folder_list.py', 'wb') as f_hand:
data_f = b"Folders = " + bytes(str(list(self.fileSys.folders)),
'utf-8')
f_hand.write(data_f)
# files list
with open(self.HOME_PATH + "/" + 'files_list.py', 'wb') as fi_hand:
data_fi = b"Files = " + bytes(str(self.fileSys.files), 'utf-8')
fi_hand.write(data_fi)
return
def cleanUpFiles(self):
sleep(.3)
folder = os.environ['USERPROFILE'].replace("\\", "/") + \
"/.musicapp" + "/_temp"
tmp_files = os.listdir(folder)
for a in tmp_files:
file = folder + "/" + a
print('Removing:', file)
os.remove(file)
def _preprocesses(self):
"""
"""
self.HOME_PATH = sys.argv[0].replace('music.py', '').replace("\\", "/")
self.main_qml = self.HOME_PATH + "/" + "ui/main.qml"
# check if user supplied music file
if len(sys.argv) > 1:
self.fileSys.prepare(sys.argv[1])
self._startUp()
def _startUp(self):
"""
"""
# start app
app = QGuiApplication(sys.argv)
app.setWindowIcon(
QIcon(self.HOME_PATH + 'ui/images/ic_album_black_24dp.png'))
engine = QQmlApplicationEngine()
self.control = Control()
engine.rootContext().setContextProperty('Functions', self.control)
engine.rootContext().setContextProperty('FileSys', self.fileSys)
engine.load(self.main_qml)
engine.quit.connect(app.quit)
app.aboutToQuit.connect(self._postprocesses)
sys.exit(app.exec_())
class Populate:
def __init__(self):
auth = dict(url=get_env("MONGO_URL", required=False),
host=get_env("MONGO_HOST", required=False),
port=get_env("MONGO_PORT", required=False),
db_name=get_env("MONGO_DBNAME"),
username=get_env("MONGO_INITDB_ROOT_USERNAME",
required=False),
password=get_env("MONGO_INITDB_ROOT_PASSWORD",
required=False))
self.target_collection_name = get_env("TARGET_COLLECTION_NAME")
self.target_collection = None
self.aux_collection_name = get_env("AUX_COLLECTION_NAME")
self.aux_collection = None
self.fs = Fs()
self.db = Db(auth)
def __enter__(self):
self.fs.connect()
self.db.connect()
def __exit__(self, _exc_type, _exc_val, _exc_tb):
if is_panic():
return
self.db.disconnect()
self.fs.disconnect()
def get_session(self):
return self.db.client.start_session()
def get_database(self, session):
return session.client[self.db.auth["db_name"]]
def get_target_collection(self, session, db):
return DbOperation(session,
db).get_collection(self.target_collection_name,
"GET TARGET COLLECTION")
def get_aux_collection(self, session, db):
return DbOperation(session,
db).get_collection(self.aux_collection_name,
"GET AUX COLLECTION")
def get_state(self):
@db_session
def _get_state(session, db):
target_collection_exists = DbOperation(
session,
db).check_collection_exists(self.target_collection_name,
"GET TARGET COLLECTION")
aux_collection_exists = DbOperation(session,
db).check_collection_exists(
self.aux_collection_name,
"GET AUX COLLECTION")
if aux_collection_exists:
return "interrupted"
else:
if target_collection_exists:
return "finished"
else:
return "clear"
return _get_state(self)
def drop_target(self):
@db_session
def _drop_target(session, db):
DbOperation(session, db).drop_collection(
self.get_target_collection(session, db),
"DROP TARGET COLLECTION")
return _drop_target(self)
def drop_aux(self):
@db_session
def _drop_aux(session, db):
DbOperation(session, db).drop_collection(
self.get_aux_collection(session, db), "DROP AUX COLLECTION")
return _drop_aux(self)
def start(self):
@db_session
def _get_entries(session, db):
target_collection = self.get_target_collection(session, db)
dummy = target_collection.find_one({"dummy": 0})
if dummy is None:
DbOperation(session, db).insert_data(target_collection,
[{
"dummy": 0
}],
"INSERT TARGET DUMMY",
use_session=False)
aux_collection = self.get_aux_collection(session, db)
return aux_collection.find().sort("tr_id", pymongo.DESCENDING)
entries = _get_entries(self)
if entries.count() == 0:
@db_session
def _delete_dummy(session, db):
target_collection = self.get_target_collection(session, db)
dummy = target_collection.find_one({"dummy": 0})
if dummy is not None:
DbOperation(session,
db).delete_data(target_collection, dummy,
"INSERT TARGET DUMMY")
_delete_dummy(self)
self.drop_aux()
return True
entry = entries[0]
entry_id, file_name, year, file_seek, header_text, tr_id = \
entry["_id"], entry["file_name"], entry["year"], entry["file_seek"], entry["header"], entry["tr_id"]
file_size = get_file_size(file_name)
print_flush(f"Populating from file '{file_name}' ({year}): ", end='')
with open(file_name, "r",
encoding=get_file_encoding(file_name)) as file:
if file_seek == 0:
entry["header"] = header_text = file.readline().strip()
entry["file_seek"] = file.tell()
@db_session
def _set_header_text(session, db):
aux_collection = self.get_aux_collection(session, db)
DbOperation(session,
db).update_data(aux_collection, entry,
"SAVE AUX HEADER")
_set_header_text(self)
else:
file.seek(file_seek)
header = strip_arr(header_text.split(';'))
header = [h.upper() for h in header]
batch_size = 1000
while True:
print_flush(
f"\rPopulating from file '{file_name}' ({year}): "
f"{format_file_size(entry['file_seek'])} / {format_file_size(file_size)} "
f"({entry['file_seek'] / file_size:.2%})",
end="")
end = False
rows = []
for i in range(batch_size):
line = []
prev_line_text = ""
while True:
line_text = prev_line_text + file.readline().strip()
if not line_text:
end = True
break
line = [
strip(l) for l in line_text.rstrip().split(';')
]
if len(line) == len(header):
break
prev_line_text = line_text
if end:
break
row = {}
for h, v in zip(header, line):
row[h] = v
row["year"] = year
rows.append(row)
@db_session
def _insert_rows(session, db):
aux_collection = self.get_aux_collection(session, db)
target_collection = self.get_target_collection(session, db)
DbOperation(session,
db).insert_data(target_collection, rows,
"INSERT ROWS")
if end:
print_flush(
f"\r\x1b[1K\rPopulating from file '{file_name}' ({year}): {' ' * 35}",
end="")
print_flush(
f"\r\x1b[1K\rPopulating from file '{file_name}' ({year}): done!"
)
DbOperation(session,
db).delete_data(aux_collection, entry,
"DROP AUX COLLECTION")
else:
entry["file_seek"] = file.tell()
entry["tr_id"] = entry["tr_id"] + 1
DbOperation(session, db).insert_data(
aux_collection, [{
"file_name": file_name,
"year": year,
"file_seek": file.tell(),
"header": header_text,
"tr_id": entry["tr_id"]
}], "UPDATE FILE SEEK")
_insert_rows(self)
@db_session
def _remove_old_aux(session, db):
aux_collection = self.get_aux_collection(session, db)
for old_entry in aux_collection.find(
({
"file_name": file_name
} if end else {
"file_name": file_name,
"tr_id": {
"$ne": entry["tr_id"]
}
})):
DbOperation(session,
db).delete_data(aux_collection, old_entry)
_remove_old_aux(self)
if end:
return self.start()
def prepare(self):
df_changed = False
for file, year in self.fs.data_files:
if get_file_encoding(file) == "":
print_flush(f"Determining encoding of file '{file}': ", end="")
read_file(file)
print_flush("done!")
df_changed = True
if df_changed:
self.fs.disconnect()
self.fs.connect()
@db_session
def _prepare(session, db):
aux_collection = self.get_aux_collection(session, db)
if len(self.fs.data_files) == 0:
return
DbOperation(session, db).insert_data(aux_collection, [{
"file_name": file,
"year": year,
"file_seek": 0,
"header": "",
"tr_id": 0
} for file, year in self.fs.data_files],
"FILL AUX COLLECTION",
use_session=False)
return False
return _prepare(self)
def do_query(self):
print_flush("Executing query...", end="")
@db_session
def _do_query(session, db):
target_collection = self.get_target_collection(session, db)
result = target_collection.aggregate([{
"$match": {
"$or": [{
"year": 2019
}, {
"year": 2020
}],
"PHYSTESTSTATUS": "Зараховано"
}
}, {
"$group": {
"_id": {
"year": "$year",
"REGNAME": "$REGNAME",
},
"max_ball": {
"$max": "$PHYSBALL100"
}
}
}])
stats = dict()
for r in result:
if r["_id"]["REGNAME"] not in stats:
stats[r["_id"]["REGNAME"]] = dict()
stats[r["_id"]["REGNAME"]][r["_id"]["year"]] = float(
r["max_ball"].replace(",", "."))
return stats
stats = _do_query(self)
with open(os.path.join(self.fs.query_folder, "query_result.csv"),
"w") as f:
f.write("Region,MaxPhysBall100_2019,MaxPhysBall100_2020\n")
for s in stats.items():
f.write(f"{s[0]},{s[1].get(2019)},{s[1].get(2020)}\n")
print_flush(" done!")