def get_bug_report(data_prefix, vectorized_data, bug_report_id):
bug_report_index_collection = UnQLite(
data_prefix + "_bug_report_index_collection_index_db")
bug_report = pickle.loads(bug_report_index_collection[bug_report_id])
bug_report_index_collection.close()
index = bug_report['report']
return vectorized_data[index, :]
def __init__(self, db_path=''):
self.db_path = db_path
self.store = UnQLite(
) #loading db from databases freezes the process!!
self.hits = 0
self.misses = 0
self.uncommited_recs = 0
class UniqLiteFactory:
def __init__(self):
self.conn = None
def connect(self):
self.conn = UnQLite(filename)
return self.conn
def disconnect(self):
self.conn.close()
def create_collection(self, name):
db = self.connect()
collection = db.collection(name)
collection.create()
print "collection " + name + " created successfully "
self.disconnect()
def drop_collection(self, name):
db = self.connect()
collection = db.collection(name)
collection.drop()
print "collection " + name + " droped successfully "
self.disconnect()
def insert(self, collection_name, data=None):
db = self.connect()
collection_name = db.collection(collection_name)
collection_name.store(data)
self.disconnect()
def getTimesFromDB(self, dbPath):
db = UnQLite(DB_PATH)
times = db.collection("times")
if not times.exists():
raise Exception("No data to retrieve")
self.dbData = times.all()
return self
class Compilation_error_db:
def __init__(self, db_path=''):
self.db_path = db_path
self.store = UnQLite() #loading db from databases freezes the process!!
self.hits = 0
self.misses = 0
self.uncommited_recs = 0
# keeping prog_id for backward compatibility
def get_errors(self, prog_id, prog):
#print("get_errors here")
if prog in self.store:
err_msg = self.store[prog]
errs = get_error_list(err_msg)
self.hits += 1
else:
errs, err_msg = compilation_errors(prog)
self.store[prog] = err_msg
self.misses += 1
self.uncommited_recs += 1
if self.uncommited_recs > 0 and self.uncommited_recs % 250 == 0:
self.commit()
return errs
def close(self):
self.store.close()
def commit(self, ):
cnt = self.uncommited_recs
self.uncommited_recs = 0
self.store.commit()
def __len__(self):
return len(self.store)
def consolidateToDB(self):
db = UnQLite(DB_PATH)
times = db.collection("times")
if not times.exists():
times.create()
for t in self.timeMap.keys():
times.store({t: self.timeMap[t]})
db.commit()
def setUp(self):
self.app = bottle.Bottle(catchall=False)
_, filename = tempfile.mkstemp(suffix='.unqlite')
self.plugin = self.app.install(unqlite.Plugin(filename=filename))
self.conn = UnQLite(filename)
self.conn.collection('todo').create()
self.conn.close()
def db(cls):
if cls._database is not None:
return cls._database
else:
if not bool(os.environ.get(_UNQLITE_READ_ONLY_ENV, False)):
cls._database = UnQLite(cls.database_file)
else:
cls._database = UnQLite(cls.database_file, flags=0x00000001)
atexit.register(cls._database.close)
return cls._database
def __init__(self, filename=None):
assert filename is not None
if not os.path.exists(filename):
directory = Path(filename[:filename.rfind('/')])
directory.mkdir(parents=True, exist_ok=True)
self.__db = UnQLite(filename)
if self.__db_get('master_password_hash'):
current_app.config['INIT_STATE'] = 2
def insert_unqlite_items(number):
db = UnQLite('tmp.unqlite')
items = db.collection('items')
items.create()
for x in xrange(number):
items.store([{
'a': str(x),
'b': '2',
'c': '3',
}])
def __init__(self, location=None):
self.location = location
if type(self.location) == str and len(self.location) > 0:
logger.debug("Connecting to database at {}".format(
os.path.abspath(location)))
self.db = UnQLite(self.location)
else:
# in-memory database
logger.debug("Creating an in-memory database.")
self.db = UnQLite()
self.collections = dict()
def get_db(source='orders', suffix='', directory=''):
col_path = source + suffix
base_path = directory
if not base_path.endswith('/'):
base_path = base_path + '/'
if not os.path.isdir(base_path):
os.mkdir(base_path)
db = UnQLite(base_path + col_path +
'.db') # Create an in-memory database.
col = db.collection(col_path)
col.create()
return col, db
def extract_enriched_api(data_prefix, bug_report_full_sha):
data = sparse.load_npz(data_prefix + '_raw_count_data.npz')
bug_report_files_collection_db = UnQLite(data_prefix +
"_bug_report_files_collection_db",
flags=0x00000100 | 0x00000001)
current_files = pickle.loads(
bug_report_files_collection_db[bug_report_full_sha])
bug_report_files_collection_db.close()
bug_report_id = bug_report_full_sha[0:7]
shas = current_files['shas']
class_name_lookup = current_files['class_name_to_sha']
bug_report_data = []
bug_report_lookup = {}
n_rows = 0
for ast_sha in shas:
ast_data, lookup, current_ast_sha = add_types_source_to_bug_report_data(
data, data_prefix, class_name_lookup, ast_sha)
current_index = n_rows
bug_report_data.append(ast_data)
for k in lookup:
lookup[k] += current_index
bug_report_lookup[current_ast_sha] = lookup
n_rows += ast_data.shape[0]
bug_report_row = get_bug_report(data_prefix, data, bug_report_id)
bug_report_data.append(bug_report_row)
bug_report_data_matrix = sparse.vstack(bug_report_data)
sparse.save_npz(
data_prefix + '_' + bug_report_id + '_partial_enriched_api',
bug_report_data_matrix)
with open(
data_prefix + '_' + bug_report_id +
'_partial_enriched_api_index_lookup', 'w') as outfile:
json.dump(bug_report_lookup, outfile)
transformer = TfidfTransformer()
tf_idf_data = transformer.fit_transform(bug_report_data_matrix)
sparse.save_npz(data_prefix + '_' + bug_report_id + '_tfidf_enriched_api',
tf_idf_data)
# print "bug_report_id", bug_report_id
return bug_report_id
def __init__(self, path, db_path, options=dict()):
self.board = Board()
self.db = UnQLite(db_path)
self.engine = Popen(path,
universal_newlines=True,
stdin=PIPE,
stdout=PIPE)
self._put('uci')
self._ready()
for option, val in options.items():
self._set_option(option, val)
self.num_games = 1
while True:
self.board.reset()
self.learn(200)
def get_user(user_uuid):
db = UnQLite(db_file)
user_col = get_collection('user')
try:
return user_col.fetch(db[user_uuid])
except KeyError:
return None
def create_demo_db(db_file, data_path, user_count, truncate=True):
if os.path.isfile(db_file):
if truncate:
os.remove(db_file)
db = UnQLite(db_file)
create_users(db, data_path, user_count)
create_tags(db, data_path)
def __init__(
self,
*,
mydomains: List[str],
local_delivery_handler: LocalDeliveryHandler,
database: UnQLite,
smtpd_auth_handler: SMTPAuthHandler,
hostname: str,
self_name: str = "mailboat.transfer_agent",
smtpd_port: int = 8025,
custom_queue: Optional[EmailQueue] = None,
) -> None:
self.mydomains = mydomains
self.database = database
self.name = self_name
self.hostname = hostname
self.queue = (custom_queue
if custom_queue else UnQLiteEmailMessageQueue(
database.collection("{}.queue".format(self_name))))
self.delivery_tasks: List[DeliveryTask] = []
self.smtpd_controller = Controller(
_SMTPDHandler(
self.handle_message,
smtp_auth_handler=smtpd_auth_handler,
),
port=smtpd_port,
hostname=hostname,
)
self.local_delivery_handler = local_delivery_handler
self._task_deliveryman = asyncio.ensure_future(
self._cothread_deliveryman())
def dump_info(dirinfo, out_path):
"""Dump FileInfo into a UnQLite database"""
out_path.unlink(missing_ok=True)
with UnQLite(out_path.as_posix()) as db:
with db.transaction():
for info in dirinfo:
db[info.name] = json.dumps(info.__dict__)
def __getitem__(self, entity) -> UnQLite:
"""
Get UnQLite index by entity name
:param entity:
:return:
"""
return UnQLite(self.uri_index_files_by_entity[entity],
flags=UNQLITE_OPEN_READONLY)
def __init__(self, filepath):
self.filepath = filepath
self.name = os.path.basename(filepath)
# In memory JSON store
self.db = UnQLite()
self.workbook = self.load_from_file(filepath)
self.data = self.extract_as_python()
self.to_db()
def get_unqlite(name="default", path=None):
'''Get unqlite object
:param name: unqlite db name, will use the name to get path from settings.UNQLITE_DB_NAMES
:param path: unqlite db file path, if None, will use configuration in get from name
'''
from uliweb import settings
from unqlite import UnQLite
if not path:
path = settings.UNQLITE_DB_NAMES.get(name)
if path == ':mem:':
#always use a same unqlite of ':mem:'
if not _v.mem_unqlite:
_v.mem_unqlite = UnQLite(path)
return _v.mem_unqlite
return UnQLite(path)
def write():
db = UnQLite(DB_FILE)
with db.transaction():
db.collection('users').create()
for line in FIRST_LINES:
with db.transaction():
stories = db.collection('stories')
stories.create()
# Are there stories that have the same first line?
same_first_line = stories.filter(
lambda story: story['lines'][0].get('text') == line)
if same_first_line:
continue
stories.store([{
"max_lines": MAX_LINES,
"locked": False,
"locked_by": None,
"locked_at": None,
"lines": [{
"text": line
}]
}])
def __init__(self, location=None):
try:
from unqlite import UnQLite
except ImportError:
raise ImportError(
"The unqlite library is required for this feature.")
self.location = location
if type(self.location) == str and len(self.location) > 0:
logger.debug("Connecting to database at {}".format(
os.path.abspath(location)))
self.db = UnQLite(self.location)
else:
# in-memory database
logger.debug("Creating an in-memory database.")
self.db = UnQLite()
self.collections = dict()
def main():
print("Start", datetime.datetime.now().isoformat())
before = default_timer()
bug_report_file_path = sys.argv[1]
print("bug report file path", bug_report_file_path)
data_prefix = sys.argv[2]
print("data prefix", data_prefix)
bug_reports = load_bug_reports(bug_report_file_path)
ast_cache_db = UnQLite(data_prefix + "_ast_cache_collection_db")
vectorize(ast_cache_db, bug_reports, data_prefix)
after = default_timer()
total = after - before
print("End", datetime.datetime.now().isoformat())
print("total time ", total)
ast_cache_db.close()
async def test_performance_local_delivery_in_pure_memory_queue(
self, unused_tcp_port: int):
TEST_MAIL_NUMBER = 100
virtual_box = []
database = UnQLite(":mem:")
async def delivery_handler(email: EmailMessage):
virtual_box.append(email)
ta = TransferAgent(
mydomains=["localhost"],
hostname="localhost",
database=database,
local_delivery_handler=delivery_handler,
smtpd_auth_handler=smtpd_auth_rejectall,
custom_queue=MemoryEmailQueue(),
smtpd_port=unused_tcp_port,
)
try:
ta.start()
email = EmailMessage()
email["Message-Id"] = "<test1@localhost>"
email["To"] = "user@localhost"
email["From"] = "qa@localhost"
t1 = perf_counter()
lost_mail_count = 0
for x in range(0, TEST_MAIL_NUMBER):
try:
await aiosmtplib.send(email,
hostname="localhost",
port=unused_tcp_port)
except:
lost_mail_count += 1
async def wait_virtual_box():
while len(virtual_box) < TEST_MAIL_NUMBER:
await asyncio.sleep(0)
await asyncio.wait_for(wait_virtual_box(), 12)
t2 = perf_counter()
result = t2 - t1
logging.warning(
"MemoryEmailQueue: %f sec./%smails, lost=%d",
result,
TEST_MAIL_NUMBER,
lost_mail_count,
)
assert result < (
TEST_MAIL_NUMBER / 100 *
4), "the MTA performance should be 25 per second at least"
finally:
ta.destory()
if len(virtual_box) != TEST_MAIL_NUMBER:
pytest.fail("except {} mails, got {}".format(
TEST_MAIL_NUMBER, len(virtual_box)))
class TestTransaction(BaseTestCase):
"""
We must use a file-based database to test the transaction functions. See
http://unqlite.org/forum/trouble-with-transactions+1 for details.
"""
def setUp(self):
self._filename = "test.db"
self.db = UnQLite(self._filename)
def tearDown(self):
try:
self.db.close()
except:
pass
if os.path.exists(self._filename):
os.unlink(self._filename)
def test_transaction(self):
@self.db.commit_on_success
def _test_success(key, value):
self.db[key] = value
@self.db.commit_on_success
def _test_failure(key, value):
self.db[key] = value
raise Exception("intentional exception raised")
_test_success("k1", "v1")
self.assertEqual(self.db["k1"], "v1")
self.assertRaises(Exception, lambda: _test_failure("k2", "v2"))
self.assertRaises(KeyError, lambda: self.db["k2"])
def test_explicit_transaction(self):
self.db.close()
self.db.open()
self.db.begin()
self.db["k1"] = "v1"
self.db.rollback()
self.assertRaises(KeyError, lambda: self.db["k1"])
class BaseTestCase(unittest.TestCase):
def setUp(self):
super(BaseTestCase, self).setUp()
self.db = UnQLite(':mem:')
self._filename = 'test.db'
self.file_db = UnQLite(self._filename)
def tearDown(self):
try:
self.file_db.close()
except:
pass
if os.path.exists(self._filename):
os.unlink(self._filename)
def store_range(self, n, db=None):
if db is None:
db = self.db
for i in range(n):
db['k%s' % i] = str(i)
class BaseTestCase(unittest.TestCase):
def setUp(self):
super(BaseTestCase, self).setUp()
self.db = UnQLite(':mem:')
self._filename = 'test.db'
self.file_db = UnQLite(self._filename)
def tearDown(self):
try:
self.file_db.close()
except:
pass
if os.path.exists(self._filename):
os.unlink(self._filename)
def store_range(self, n, db=None):
if db is None:
db = self.db
for i in range(n):
db['k%s' % i] = str(i)
def __init__(self, brpath, dbpath):
self.br = Engine(brpath)
# f*****g cute chess
self.br.write('setoption name Threads value 3')
self.br.write('setoption name Hash value 4096')
# f*****g cute chess
self.db = UnQLite(dbpath)
self.board = Board()
self.func_book = {
'uci': self.uci,
'isready': self.isready,
'go': self.go,
'quit': self.quit,
'position': self.position
}
self.hello()
try:
self.start()
except KeyboardInterrupt:
self.quit(None)
def get_diff(dirinfo, db_path):
"""Yield entries that are different in the database (or not in it)"""
with UnQLite(db_path.as_posix()) as db:
for info in dirinfo:
if info.name in db:
dbinfo = json.loads(db[info.name])
if (info.size != dbinfo['size']
and info.mtime != dbinfo['mtime']
and info.md5 != dbinfo['md5']):
yield info
else:
yield info
def process(bug_reports, repository_path, data_prefix):
ast_cache = prepare_ast_cache(repository_path)
ast_cache_collection_db = UnQLite(data_prefix + "_ast_cache_collection_db")
before = default_timer()
for k, v in ast_cache.items():
ast_cache_collection_db[k] = pickle.dumps(v, -1)
after = default_timer()
total = after - before
print("total ast cache saving time ", total)
bug_report_files = prepare_bug_report_files(repository_path, bug_reports, ast_cache)
before = default_timer()
bug_report_files_collection_db = UnQLite(data_prefix + "_bug_report_files_collection_db")
for k, v in bug_report_files.items():
bug_report_files_collection_db[k] = pickle.dumps(v, -1)
after = default_timer()
total = after - before
print("total bug report files saving time ", total)
def __init__(self, master, local):
"""define UnQLite database"""
db = UnQLite()
""" define collection where we'll insert local files content"""
self.db_local = db.collection("db_local")
self.db_local.create()
""" define collection where we'll insert master files content"""
self.db_master = db.collection("db_master")
self.db_master.create()
with open(local) as bibtex_file:
bibtex_str = bibtex_file.read()
"""create bibtex database for local file"""
self.bibdb_local = bibtexparser.loads(bibtex_str)
self.db_local.store(self.bibdb_local.entries)
with open(master) as bibtex_file:
bibtex_str = bibtex_file.read()
"""create bibtex database for master file"""
bibdb_master = bibtexparser.loads(bibtex_str)
self.db_master.store(bibdb_master.entries)
class UnQDb:
"""
基于UnQLite的nosql数据存贮
"""
def __init__(self, dbpath='data.db'):
# UnQLite.__init__(self,dbpath)
self.db = UnQLite(dbpath)
self.dbpath = dbpath
def __del__(self):
self.db.close()
def add(self, key, value):
"""
添加数据
key ='2eas'
value={}
"""
self.db[key] = value
# print('222')
def reload(self):
self.db = UnQLite(self.dbpath)
pass
def get(self, key):
"""
获取数据
自动转换成字典
"""
# print('value',value)
try:
value = str(self.db[key], "utf-8")
value = ast.literal_eval(value)
except:
return None
pass
return value
def get_all(self):
with self.db.cursor() as cursor:
for key, value in cursor:
yield key, self.get(key)
def delete(self, key):
"""
删除数据
"""
# del self.db[key]
self.db.delete(key)
def col(self, key):
self.col = self.db.collection(key)
self.col.create() # Create the collection if it does not exist.
self.col.exists()
return self.col
def __init__( self, configFileName, dbPath, timeout ):
self.dnsArray = list()
self.responseTimes = dict()
if os.access( configFileName, os.R_OK ) is False:
print "Can not read "+configFileName+" !"
exit(-1)
if os.access( dbPath, os.F_OK ) is False:
print "Created DB Directory at "+dbPath
os.mkdir( dbPath )
db = UnQLite( dbPath+"/history.db" )
self.dnsDB = db.collection('dnsResponses')
if ( self.dnsDB.exists() is False ):
print "Initial DB Created."
self.dnsDB.create()
self.recursor = dns.resolver.Resolver()
self.recursor.lifetime = timeout
self.recursor.timeout = self.recursor.lifetime
dns.resolver.override_system_resolver(self.recursor)
with open(configFileName) as configFile:
contents = configFile.readlines()
validNSRegex = re.compile(r'nameserver (1[0-9][0-9]|2[0-4][0-9]|25[0-5]|[0-9][0-9]|[0-9])\.(1[0-9][0-9]|2[0-4][0-9]|25[0-5]|[0-9][0-9]|[0-9])\.(1[0-9][0-9]|2[0-4][0-9]|25[0-5]|[0-9][0-9]|[0-9])\.(1[0-9][0-9]|2[0-4][0-9]|25[0-5]|[0-9][0-9]|[0-9])')
for line in contents:
match = validNSRegex.search(line)
if ( match is not None ):
self.dnsArray.append(match.group(1)+"."+match.group(2)+"."+match.group(3)+"."+match.group(4))
print "=> Added "+match.group(1)+"."+match.group(2)+"."+match.group(3)+"."+match.group(4)+" into the list."
self.currentPreferredDNS = self.dnsArray[0]
return
def query_by_field_intersect(db: UnQLite, col: str, key: Hashable,
value: List[Any]) -> List[dict]:
script = """
$zCallback = function($doc) {
return count(array_intersect($doc[$key], $value)) > 0;
};
$data = db_fetch_all($col, $zCallback);
"""
with db.vm(script) as vm:
vm["col"] = col
vm["key"] = key
vm["value"] = value
vm.execute()
return vm["data"]
def create_user(name, mail, description, tags, active=False, is_admin=False):
user_col = get_collection('user')
user_uuid = gen_uuid()
user = {
'id': user_uuid,
'name': name,
'mail': mail,
'description': description,
'tags': tags,
'active': active,
'is_admin': is_admin
}
user_id = user_col.store(user)
db = UnQLite(db_file)
db[user_uuid] = user_id
return user_uuid
def query_by_field_eq(db: UnQLite, col: str, key: Hashable,
value: Any) -> List[dict]:
script = """
$zCallback = function($doc) {
return $doc[$key] == $value;
};
$data = db_fetch_all($col, $zCallback);
"""
with db.vm(script) as vm:
vm["col"] = col
vm["key"] = key
vm["value"] = value
vm.execute()
return vm["data"]
class XlsxReader(object):
def __init__(self, filepath):
self.filepath = filepath
self.name = os.path.basename(filepath)
# In memory JSON store
self.db = UnQLite()
self.workbook = self.load_from_file(filepath)
self.data = self.extract_as_python()
self.to_db()
def __repr__(self):
return '<XlsxReader: {}>'.format(self.name)
@staticmethod
def load_from_file(filepath):
# read_only: optimised read, data_only: ignores formulae
return load_workbook(filename=filepath, read_only=True, data_only=True)
def extract_as_python(self):
sheets = self.workbook.worksheets
sheet_data = {}
columns_data = {}
for sheet in sheets:
sheet_title = sheet.title
rows, columns = self.extract_sheet(sheet)
sheet_data[sheet_title] = rows
columns_data[sheet_title] = columns
sheet_data.update({'_columns': [columns_data]})
return sheet_data
def extract_sheet(self, sheet):
rows = sheet.rows
# Assume the first row of the sheet is the columns
column_cells = next(rows)
columns = [cell.value for cell in column_cells]
extracted_rows = []
for row_idx, row in enumerate(rows, start=1):
row_dict = {'row': row_idx}
# Index into the columns list.
# Assumes the order is always the same.
for column_idx, cell in enumerate(row):
cell_value = self.extract_cell(cell)
row_dict[columns[column_idx]] = cell_value
extracted_rows.append(row_dict)
return extracted_rows, columns
def extract_cell(self, cell):
"""
Workarounds for OpenPyXL live here.
If the cell value has a comma, we assume it is a list and split it.
All dates in Excel are TZ naive, so add one.
"""
# TODO: test me directly
val = cell.value
if isinstance(val, unicode):
# Takes 'foo', bar'
# Returns ['foo', 'bar']
if ',' in val:
return [i.lstrip() for i in val.split(',')]
if isinstance(val, datetime):
# Just assume everything is UTC for the time being.
return UTC.localize(val)
return val
def to_json(self):
return json.dumps(self.data, default=json_serialiser)
def to_db(self):
self.collections = {
name: self.create_collection(name, documents)
for name, documents in self.data.items()
}
def create_collection(self, name, documents):
collection = self.db.collection(name)
collection.create()
docs = self.pre_store(documents)
collection.store(docs)
return collection
def pre_store(self, documents):
"""
We can't change UnQLite's json serialiser, so we need to
check for datetime objs here. We drop the 'row' key as it's of
no use once stored.
Also, we try to naively call json.dumps() to raise errors,
since UnQLite silently stores a None if it can't serialise.
Doh!
"""
checked_docs = []
for doc in documents:
checked_doc = {}
for k, v in doc.items():
if k == 'row':
continue
if isinstance(v, datetime):
checked_doc[k] = v.isoformat()
continue
json.dumps(v)
checked_doc[k] = v
checked_docs.append(checked_doc)
return checked_docs
def setUp(self):
super(BaseTestCase, self).setUp()
self.db = UnQLite(':mem:')
self._filename = 'test.db'
self.file_db = UnQLite(self._filename)
__author__ = 'James Veitch'
'''
Example to connect and download all issues on the remote server to a local cache nosql
database using UnQLite
'''
from unqlite import UnQLite
from pprint import pprint
import pythreesixty.core.mobile as mobile
db = UnQLite() # Creates an in-memory database.
issues = db.collection('issues')
issues.create() # Create the collection if it does not exist.
projects = mobile.projects()
print("User has access to {count} projects".format(count=len(projects.json())))
for p in projects.json():
print("\n{id}: {name}".format(
id=p['project_id'],
name=p['name'])
)
# Get issues
issues_request = mobile.get_issues(p['project_id'], count_only=True) # use count_only to just return the number
if issues_request.status_code == 200:
print(" - {count} issues found.".format(count=issues_request.json()['count']))
issues.store(mobile.get_issues(p['project_id']).json())
# -*- coding: utf-8 -*-
import hashlib
import os
from unqlite import UnQLite
import exifread
import datetime
db = UnQLite('./memories.db')
pictures = db.collection('pictures')
pictures.create()
pictures.store({'name': 'Leslie', 'color': 'also green'})
print pictures.fetch(0)
if __name__ == '__main__':
for dirpath, dirs, files in os.walk('/home/stephan/Images/2015'):
for filename in files:
with open(os.path.join(dirpath, filename), 'rb') as fname:
exif = exifread.process_file(fname, details=False)
photo_date = datetime.datetime.strptime(exif['EXIF DateTimeOriginal'].values, '%Y:%m:%d %H:%M:%S')
resolution = (exif['EXIF ExifImageWidth'].values[0], exif['EXIF ExifImageLength'].values[0])
inserted = modified = datetime.datetime.now()
md5 = hashlib.md5(fname.read()).hexdigest()
print filename
print photo_date
print resolution
print inserted
print modified
print md5
parser = argparse.ArgumentParser(description='Ubicajeros API actualizar db')
parser.add_argument('-r', '--radio', required=False, help='Radio de busqueda en kms.', default='1000')
parser.add_argument('-l', '--latlon', required=False, help='Latlon de busqueda', default='19.432608,-99.133209')
args = parser.parse_args()
CAJEROS_URL = 'http://www.banxico.org.mx/consultas-atm/cajeros.json?l=' + args.latlon + '&b=&r=' + args.radio
CAJERO_URL = 'http://www.banxico.org.mx/consultas-atm/cajeros/info.json'
print 'Buscando cajeros en ' + CAJEROS_URL
cajeros_json = requests.get(CAJEROS_URL).json()['contenido']
total_cajeros = len(cajeros_json)
agregados = 0
for i, cajero_json in enumerate(cajeros_json):
db = UnQLite('cajeros.db')
db.open()
cajero = {}
cajero['id'] = cajero_json['id']
cajero['clave_institucion'] = cajero_json['cb']
cajero['lat'] = cajero_json['l']['lat']
cajero['lon'] = cajero_json['l']['lng']
cajero['nombre_institucion'] = NOMBRES[cajero['clave_institucion']]
try:
print 'Cajero ' + str(i) + ' de ' + str(total_cajeros) + ', ' + str(cajero['id']) + ' existe? ' + str(db.exists(cajero['id']))
if not db.exists(cajero['id']):
url_cajero = CAJERO_URL + '?id=' + str(cajero['id']) + '&banco=' + str(cajero['clave_institucion'])
cajero_json = requests.get(url_cajero).json()['contenido']
cajero['cp'] = str(cajero_json['cp'])
cajero['horario'] = cajero_json['hs']
cajero['direccion'] = cajero_json['d']
def setUp(self):
self._filename = "test.db"
self.db = UnQLite(self._filename)