def read_local_conf(conf_file=None):
if not conf_file:
home = os.path.expanduser('~')
conf_file = os.path.join(home, '.photolog')
log.info('Reading config file: %s' % conf_file)
conf = yaml.load(open(conf_file))
return conf
def upload_directories(targets, filelist, host, secret, tags, skip, halt):
start = time()
first_batch, second_batch = [], []
for target in targets:
if os.path.isdir(target):
for file in os.listdir(target):
name, ext = os.path.splitext(file)
ext = ext.lstrip('.').lower()
if ext not in ALLOWED_FILES:
continue
full_file = os.path.join(target, file)
if ext in IMAGE_FILES:
first_batch.append((file, full_file))
elif ext in RAW_FILES:
second_batch.append((file, full_file))
else:
name, ext = os.path.splitext(target)
ext = ext.lstrip('.').lower()
full_file = os.path.abspath(target)
if ext not in ALLOWED_FILES:
continue
if ext in IMAGE_FILES:
first_batch.append((target, full_file))
elif ext in RAW_FILES:
second_batch.append((target, full_file))
for target in filelist:
name, ext = os.path.splitext(target)
ext = ext.lstrip('.').lower()
full_file = os.path.abspath(target)
if ext not in ALLOWED_FILES:
continue
if ext in IMAGE_FILES:
first_batch.append((target, full_file))
elif ext in RAW_FILES:
second_batch.append((target, full_file))
n, skipped = 1, 0
total_files = len(first_batch) + len(second_batch)
log.info('Found %s files' % total_files)
for batch in chunks(sorted(first_batch) + sorted(second_batch), BATCH_SIZE):
#batch_id = start_batch(endpoint, secret)
for file, full_file in batch:
log.info('Uploading %s [%s/%s]' % (full_file, n, total_files))
file_start = time()
uploaded = handle_file(host, full_file, secret, tags, skip, halt)
skipped += 1 if not uploaded else 0
pct = 100 * n / total_files
log.info("Done in %0.2fs [%0.1f%%]" % (time() - file_start, pct))
n += 1
elapsed = time() - start
log.info('Skipped files: %s' % skipped)
log.info('Uploaded %s files in %.2fs' % (total_files, elapsed))
def handle_file(host, full_file, secret, tags, skip, halt):
"""
:param host: Host to upload data to
:param full_file: Full file path in local machine
:param secret: API secret
:param tags: Tags to use for file
:param skip: Steps for job to skip
:param halt: If True, will wait for user input to resume after attempts
:return: Returns if the file was uploaded or not
"""
answer = 'Y'
while answer == 'Y':
attempt = 1
while attempt < UPLOAD_ATTEMPTS:
try:
validate_file(full_file)
file_exists = verify_exists(host, full_file, secret)
endpoint = urljoin(host, '/photos/')
if file_exists:
log.info('File %s already uploaded' % full_file)
return False
else:
response = requests.post(endpoint, data={
'tags': tags,
'skip': skip,
# 'batch_id': None,
# 'is_last': False, # n == total_files
}, files={
'photo_file': open(full_file, 'rb'),
}, headers={
'X-PHOTOLOG-SECRET': secret
})
return response.status_code == 201
except requests.ConnectionError:
attempt += 1
log.warning("Attempt %s. Failed to connect. Retrying" % attempt)
except OSError:
log.warning("Invalid file: %s - Skipping" % full_file)
return False
if halt:
answer = input("Problem connecting, Continue? [Y, n]") or 'Y'
else:
answer = 'n'
raise requests.ConnectionError('Could not connect to %s' % host)
